From 06d278a402b8a2ebd3f7e8247e28051e8de6609f Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 14 Oct 2026 14:22:46 +0800 Subject: [PATCH 1/5] fix compile --- be/src/exprs/function/cast/cast_to_variant.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/be/src/exprs/function/cast/cast_to_variant.h b/be/src/exprs/function/cast/cast_to_variant.h index 8b95aca904fbfe..0b5731e1c1cb58 100644 --- a/be/src/exprs/function/cast/cast_to_variant.h +++ b/be/src/exprs/function/cast/cast_to_variant.h @@ -180,8 +180,8 @@ struct CastToVariant { }; // create corresponding variant value to wrap from_type -WrapperType create_cast_to_variant_wrapper(const DataTypePtr& from_type, - const DataTypeVariant& to_type) { +inline WrapperType create_cast_to_variant_wrapper(const DataTypePtr& from_type, + const DataTypeVariant& to_type) { if (from_type->get_primitive_type() == TYPE_VARIANT) { // variant_max_subcolumns_count is not equal return create_unsupport_wrapper(from_type->get_name(), to_type.get_name()); @@ -190,8 +190,8 @@ WrapperType create_cast_to_variant_wrapper(const DataTypePtr& from_type, } // create corresponding type convert from variant -WrapperType create_cast_from_variant_wrapper(const DataTypeVariant& from_type, - const DataTypePtr& to_type) { +inline WrapperType create_cast_from_variant_wrapper(const DataTypeVariant& from_type, + const DataTypePtr& to_type) { if (to_type->get_primitive_type() == TYPE_VARIANT) { // variant_max_subcolumns_count is not equal return create_unsupport_wrapper(from_type.get_name(), to_type->get_name()); From 441a4ddb6b357db0f4b18288f352fd27d90ea64b Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Thu, 15 Oct 2026 13:05:44 +0800 Subject: [PATCH 2/5] paimon jni scanner --- be/src/exec/scan/access_path_parser.cpp | 4 ++ be/src/exec/scan/file_scanner.cpp | 11 +++ be/src/format/jni/jni_data_bridge.cpp | 54 +++++++++++++++ .../doris/paimon/PaimonColumnValue.java | 67 ++++++++++++++++++- .../apache/doris/paimon/PaimonJniScanner.java | 4 ++ .../paimon/PaimonReadTypeProjection.java | 6 ++ .../paimon/PaimonReaderOptions.java | 8 ++- .../doris/datasource/paimon/PaimonUtil.java | 3 + .../paimon/source/PaimonScanNode.java | 9 ++- 9 files changed, 160 insertions(+), 6 deletions(-) diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index d427ba6b76e9e0..5f1bdcb06f8e1c 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -439,6 +439,10 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, } switch (nested_type->get_primitive_type()) { + case TYPE_FILE: + // FILE has a fixed six-child schema. Keep the whole value in the scan projection; + // element_at selects its public field after the reader materializes the FILE column. + return Status::OK(); case TYPE_STRUCT: return build_struct_children_from_access_node( column, assert_cast(*nested_type), node, path, schema_column, diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index ee5203f16f0930..2799d56e5c458e 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -44,6 +44,7 @@ #include "core/column/column_nullable.h" #include "core/column/column_vector.h" #include "core/data_type/data_type.h" +#include "core/data_type/data_type_file.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_string.h" #include "core/string_ref.h" @@ -1055,6 +1056,16 @@ Status FileScanner::_get_next_reader() { const TFileRangeDesc& range = _current_range; _current_range_path = range.path; + if (range.__isset.table_format_params && range.table_format_params.__isset.paimon_params && + (range.table_format_params.paimon_params.__isset.blob_files || + std::ranges::any_of(_file_slot_descs, [](const SlotDescriptor* slot) { + return contains_file_type(slot->get_data_type_ptr()); + }))) { + return Status::NotSupported( + "Paimon BLOB columns require enable_file_scanner_v2=true for both native " + "and JNI readers"); + } + if (!_partition_slot_descs.empty()) { // we need get partition columns first for runtime filter partition pruning RETURN_IF_ERROR(_generate_partition_columns()); diff --git a/be/src/format/jni/jni_data_bridge.cpp b/be/src/format/jni/jni_data_bridge.cpp index cd4f20b0bbdd63..871016d9dd9567 100644 --- a/be/src/format/jni/jni_data_bridge.cpp +++ b/be/src/format/jni/jni_data_bridge.cpp @@ -24,6 +24,7 @@ #include "core/block/block.h" #include "core/column/column_array.h" +#include "core/column/column_file.h" #include "core/column/column_map.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" @@ -31,6 +32,7 @@ #include "core/column/column_varbinary.h" #include "core/column/variant_v2/column_variant_v2.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_file.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" @@ -44,6 +46,34 @@ namespace doris { +namespace { + +// FILE keeps its Doris type in the destination block. The JNI wire layout is the same +// six-child layout as STRUCT, so Java can reuse VectorColumn's existing nested transport. +std::string jni_file_type(const DataTypePtr& type, bool encoded_names) { + const auto& file_type = assert_cast(*remove_nullable(type)); + std::ostringstream buffer; + buffer << "struct<"; + for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) { + if (i != 0) { + buffer << ","; + } + if (encoded_names) { + std::string encoded_name; + base64_encode(file_type.get_element_name(i), &encoded_name); + buffer << "$" << encoded_name; + } else { + buffer << file_type.get_element_name(i); + } + // Type lengths remain authoritative on BE; Java only needs the binary value layout. + buffer << ":" << JniDataBridge::get_jni_type(file_type.get_element(i)); + } + buffer << ">"; + return buffer.str(); +} + +} // namespace + #define FOR_FIXED_LENGTH_TYPES(M) \ M(PrimitiveType::TYPE_TINYINT, ColumnInt8, Int8) \ M(PrimitiveType::TYPE_BOOLEAN, ColumnUInt8, UInt8) \ @@ -146,6 +176,15 @@ Status JniDataBridge::fill_column(TableMetaAddress& address, ColumnPtr& doris_co case PrimitiveType::TYPE_STRUCT: status = _fill_struct_column(address, data_column, data_type, num_rows); break; + case PrimitiveType::TYPE_FILE: { + auto& file = assert_cast(*data_column); + const auto& file_type = assert_cast(*remove_nullable(data_type)); + for (size_t i = 0; i < DataTypeFile::FIELD_COUNT && status.ok(); ++i) { + status = fill_column(address, file.get_column_ptr(i), file_type.get_element(i), + num_rows); + } + break; + } case PrimitiveType::TYPE_VARBINARY: status = _fill_varbinary_column(address, data_column, num_rows); break; @@ -297,6 +336,8 @@ std::string JniDataBridge::get_jni_type(const DataTypePtr& data_type) { DataTypePtr type = remove_nullable(data_type); std::ostringstream buffer; switch (type->get_primitive_type()) { + case TYPE_FILE: + return jni_file_type(data_type, false); case TYPE_BOOLEAN: return "boolean"; case TYPE_TINYINT: @@ -403,6 +444,8 @@ std::string JniDataBridge::get_jni_type_with_different_string(const DataTypePtr& DataTypePtr type = remove_nullable(data_type); std::ostringstream buffer; switch (data_type->get_primitive_type()) { + case TYPE_FILE: + return jni_file_type(data_type, false); case TYPE_BOOLEAN: return "boolean"; case TYPE_TINYINT: @@ -535,6 +578,8 @@ std::string JniDataBridge::encode_string_list(const std::vector& va std::string JniDataBridge::get_jni_type_with_encoded_struct_fields(const DataTypePtr& data_type) { switch (data_type->get_primitive_type()) { + case TYPE_FILE: + return jni_file_type(data_type, true); case TYPE_STRUCT: { const auto* type_struct = assert_cast(remove_nullable(data_type).get()); @@ -634,6 +679,15 @@ Status JniDataBridge::_fill_column_meta(const ColumnPtr& doris_column, const Dat } break; } + case PrimitiveType::TYPE_FILE: { + const auto& file = assert_cast(*data_column); + const auto& file_type = assert_cast(*remove_nullable(data_type)); + for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) { + RETURN_IF_ERROR( + _fill_column_meta(file.get_column_ptr(i), file_type.get_element(i), meta_data)); + } + break; + } case PrimitiveType::TYPE_MAP: { const auto& map = assert_cast(*data_column); const auto& key_type = diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index f8efad58e80995..a5264642a7ddf6 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -20,14 +20,20 @@ import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.ColumnValue; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.data.DataGetters; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.variant.Variant; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BlobType; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; @@ -48,6 +54,15 @@ public class PaimonColumnValue implements ColumnValue { private static final Logger LOG = LoggerFactory.getLogger(PaimonColumnValue.class); private static final Map DORIS_TIME_ZONE_ALIASES; + // JNI transports Doris FILE as a six-child STRUCT. This is only the adapter's + // child schema; the Paimon reader must continue to read the logical BLOB type. + private static final RowType FILE_ROW_TYPE = DataTypes.ROW( + DataTypes.FIELD(0, "uri", DataTypes.STRING()), + DataTypes.FIELD(1, "offset", DataTypes.BIGINT()), + DataTypes.FIELD(2, "size", DataTypes.BIGINT()), + DataTypes.FIELD(3, "content_type", DataTypes.STRING()), + DataTypes.FIELD(4, "checksum", DataTypes.STRING()), + DataTypes.FIELD(5, "inline", DataTypes.BYTES())); static { Map aliases = new HashMap<>(ZoneId.SHORT_IDS); @@ -69,6 +84,8 @@ public class PaimonColumnValue implements ColumnValue { private PaimonVariantProjection variantProjection; // rebuild subpath variant to doris private Variant materializedVariant; + private boolean blobAsDescriptor; + private GenericRow materializedBlob; // Keep these caches lazy so scalar columns do not pay for complex-type reuse bookkeeping. private List arrayValues; private List mapKeys; @@ -100,6 +117,7 @@ public void setIdx( ColumnType dorisType, DataType dataType, PaimonVariantProjection variantProjection) { + clearMaterializedBlob(); this.idx = idx; this.dorisType = dorisType; this.dataType = dataType; @@ -108,6 +126,7 @@ public void setIdx( } public void setOffsetRow(InternalRow record) { + clearMaterializedBlob(); this.record = record; this.materializedVariant = null; } @@ -116,6 +135,10 @@ public void setTimeZone(String timeZone) { this.timeZone = resolveTimeZone(timeZone); } + public void setBlobAsDescriptor(boolean blobAsDescriptor) { + this.blobAsDescriptor = blobAsDescriptor; + } + @Override public boolean canGetStringAsBytes() { return true; @@ -279,9 +302,16 @@ public void unpackMap(List keys, List values) { @Override public void unpackStruct(List structFieldIndex, List values) { - RowType rowType = (RowType) dataType; - // Projection entries are original child indexes, so the binary row must keep the full RowType arity. - InternalRow row = record.getRow(idx, rowType.getFieldCount()); + RowType rowType; + InternalRow row; + if (dataType instanceof BlobType) { + rowType = FILE_ROW_TYPE; + row = getBlobFileRow(); + } else { + rowType = (RowType) dataType; + // Projection entries are original child indexes, so keep the full RowType arity. + row = record.getRow(idx, rowType.getFieldCount()); + } if (structValues == null) { structValues = new ArrayList<>(); } @@ -291,6 +321,34 @@ public void unpackStruct(List structFieldIndex, List value } } + private InternalRow getBlobFileRow() { + if (materializedBlob == null) { + materializedBlob = new GenericRow(FILE_ROW_TYPE.getFieldCount()); + } + Blob blob = record.getBlob(idx); + if (blobAsDescriptor) { + // Use the SDK's typed descriptor API. Payload bytes are never inspected to guess + // whether they contain a descriptor, and this path does not read the payload. + BlobDescriptor descriptor = blob.toDescriptor(); + materializedBlob.setField(0, BinaryString.fromString(descriptor.uri())); + materializedBlob.setField(1, descriptor.offset()); + materializedBlob.setField(2, descriptor.length() < 0 ? null : descriptor.length()); + } else { + materializedBlob.setField(5, blob.toData()); + } + return materializedBlob; + } + + private void clearMaterializedBlob() { + if (materializedBlob != null) { + // VectorColumn consumes children synchronously. Drop the prior payload as soon as + // this wrapper advances, rather than retaining a large byte array across columns. + for (int i = 0; i < FILE_ROW_TYPE.getFieldCount(); i++) { + materializedBlob.setField(i, null); + } + } + } + private PaimonColumnValue reuseColumnValue( List cache, int cacheIndex, DataGetters childRecord, int childIndex, ColumnType childDorisType, DataType childPaimonType) { @@ -306,11 +364,13 @@ private PaimonColumnValue reuseColumnValue( // VectorColumn consumes unpacked values synchronously, before this parent advances to another value. value.reset(childRecord, childIndex, childDorisType, childPaimonType, timeZone); } + value.blobAsDescriptor = blobAsDescriptor; return value; } private void reset( DataGetters record, int idx, ColumnType dorisType, DataType dataType, ZoneId timeZone) { + clearMaterializedBlob(); this.record = record; this.idx = idx; this.dorisType = dorisType; @@ -336,5 +396,6 @@ private void clearChildCaches() { mapKeys = null; mapValues = null; structValues = null; + materializedBlob = null; } } diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 9169e4ac8d2321..8e41475525922b 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -29,6 +29,7 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.disk.IOManagerImpl; import org.apache.paimon.options.MemorySize; +import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.DelegatedFileStoreTable; @@ -194,6 +195,9 @@ public void open() throws IOException { } private void initReader() throws IOException { + // Read the same effective option that the SDK uses, including table-cache hits and + // relation-level overrides carried in serialized_table. + columnValue.setBlobAsDescriptor(new Options(table.options()).get(CoreOptions.BLOB_AS_DESCRIPTOR)); ReadBuilder readBuilder = table.newReadBuilder(); if (this.fields.length > this.paimonAllFieldNames.size()) { throw new IOException( diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java index 6304bfdabbe662..88393c50c6a35a 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java @@ -20,6 +20,7 @@ import org.apache.doris.common.jni.vec.ColumnType; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BlobType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.MapType; @@ -40,6 +41,11 @@ private PaimonReadTypeProjection() { } static DataType project(DataType tableType, ColumnType requiredType) { + // FILE uses STRUCT on the JNI wire, but its six metadata children are constructed + // after reading BLOB. Asking Paimon to project those children would change its schema. + if (tableType instanceof BlobType) { + return tableType; + } if (requiredType.isStruct()) { return projectRow(tableType, requiredType); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java index 5a422f6af5a9f2..2ccf62b0d89375 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java @@ -37,7 +37,7 @@ import java.util.OptionalInt; import java.util.Set; -/** Validation shared by catalog-scoped and relation-scoped Paimon reader tuning. */ +/** Validation shared by catalog-scoped and relation-scoped Paimon reader options. */ public final class PaimonReaderOptions { public static final String TABLE_OPTION_PREFIX = "paimon.table-option."; public static final int MIN_READ_BATCH_SIZE = 1; @@ -47,9 +47,10 @@ public final class PaimonReaderOptions { public static final long MIN_ASYNC_THRESHOLD_BYTES = 1024L * 1024L; public static final long MAX_ASYNC_THRESHOLD_BYTES = 1024L * 1024L * 1024L; - // Keep this list to batch-read controls consumed by Doris' Paimon scan path. Context selectors, + // Keep this list to reader options consumed by Doris' Paimon scan path. Context selectors, // streaming-source settings, storage layout, and write options are unsafe after schema binding. private static final Set SUPPORTED_OPTIONS = ImmutableSet.of( + CoreOptions.BLOB_AS_DESCRIPTOR.key(), CoreOptions.READ_BATCH_SIZE.key(), CoreOptions.FILE_READER_ASYNC_THRESHOLD.key(), CoreOptions.FILE_INDEX_READ_ENABLED.key(), @@ -61,6 +62,7 @@ public final class PaimonReaderOptions { // These settings do not alter the selected snapshot or manifest projection, so relation-local // copies can reuse the memoized partition projection while still planning splits from the copy. private static final Set METADATA_NEUTRAL_OPTIONS = ImmutableSet.of( + CoreOptions.BLOB_AS_DESCRIPTOR.key(), CoreOptions.READ_BATCH_SIZE.key(), CoreOptions.FILE_READER_ASYNC_THRESHOLD.key(), CoreOptions.FILE_INDEX_READ_ENABLED.key(), @@ -104,6 +106,8 @@ public static void validate(String key, String value) { parse(key, value, CoreOptions.SOURCE_SPLIT_OPEN_FILE_COST); } else if (CoreOptions.SCAN_MANIFEST_PARALLELISM.key().equals(key)) { validateManifestParallelism(value); + } else if (CoreOptions.BLOB_AS_DESCRIPTOR.key().equals(key)) { + parse(key, value, CoreOptions.BLOB_AS_DESCRIPTOR); } else if (CoreOptions.FILE_INDEX_READ_ENABLED.key().equals(key)) { parse(key, value, CoreOptions.FILE_INDEX_READ_ENABLED); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index a35067837076f1..8387e8f7495e5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.PartitionValue; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.FileType; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; @@ -336,6 +337,8 @@ private static Type paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataT // Paimon VarBinaryType length is in [1, 2147483647] int varbinaryLen = ((VarBinaryType) dataType).getLength(); return ScalarType.createVarbinaryType(varbinaryLen); + case BLOB: + return FileType.create(); case DECIMAL: DecimalType decimal = (DecimalType) dataType; return ScalarType.createDecimalV3Type(decimal.getPrecision(), decimal.getScale()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 111bbc905c0281..423976177d56de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -1021,6 +1021,7 @@ && splitHasOrcFile((DataSplit) split) && nativeSplit && !fallbackRead && !isFullyMaterializedPkDvSplit((DataSplit) split); boolean canUseRust = sessionVariable.isEnablePaimonRustReader() + && !sessionVariable.isForceJniScanner() && !hasProjectedFileColumn() && sessionVariable.enableFileScannerV2 && nativeSplit && !fallbackRead && !isIncremental && providerModeTranslatable && !queryAuthTable && !restTokenTable @@ -1156,7 +1157,9 @@ protected List getDeleteFiles(TFileRangeDesc rangeDesc) { @Override public List getSplits(int numBackends) throws UserException { - boolean forceJniScanner = sessionVariable.isForceJniScanner(); + // BLOB columns are materialized as FILE by the JNI adapter. Keep their logical + // Paimon splits intact instead of converting them to independent raw-file reads. + boolean forceJniScanner = sessionVariable.isForceJniScanner() || hasProjectedFileColumn(); // Paimon system tables need Paimon-side semantics: // - binlog: pack/merge + array materialization // - audit_log: rowkind / sequence-number projection @@ -1347,6 +1350,10 @@ Map getBackendPaimonOptions() { return backendOptions; } + private boolean hasProjectedFileColumn() { + return desc.getSlots().stream().anyMatch(slot -> slot.getType().typeContainsFile()); + } + @VisibleForTesting boolean shouldForceJniForSystemTable() { if (source == null) { From a7da26c5380db7d5aa3e5526b1517b896a8d95f1 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Fri, 16 Oct 2026 14:26:03 +0800 Subject: [PATCH 3/5] support read paimon blob column --- be/src/core/data_type/data_type_file.h | 5 + be/src/format_v2/blob/blob_file_reader.cpp | 244 ++++++++++ be/src/format_v2/blob/blob_file_reader.h | 61 +++ .../blob/paimon_blob_column_reader.cpp | 193 ++++++++ .../blob/paimon_blob_column_reader.h | 95 ++++ .../blob/paimon_blob_materializer.cpp | 116 +++++ .../format_v2/blob/paimon_blob_materializer.h | 67 +++ be/src/format_v2/column_mapper.cpp | 9 + be/src/format_v2/column_mapper.h | 7 + be/src/format_v2/jni/paimon_jni_reader.cpp | 4 +- be/src/format_v2/table/iceberg_reader.cpp | 1 + be/src/format_v2/table/paimon_reader.cpp | 68 ++- be/src/format_v2/table/paimon_reader.h | 15 +- .../format_v2/blob/blob_file_reader_test.cpp | 356 ++++++++++++++ be/test/format_v2/blob/blob_test_util.h | 84 ++++ .../blob/paimon_blob_column_reader_test.cpp | 207 ++++++++ .../blob/paimon_blob_materializer_test.cpp | 222 +++++++++ .../format_v2/blob/paimon_blob_test_util.h | 157 +++++++ .../paimon/run15.sql | 262 +++++++++++ .../doris/paimon/PaimonColumnValue.java | 32 +- .../apache/doris/paimon/PaimonJniScanner.java | 19 +- .../paimon/source/PaimonScanNode.java | 219 ++++++++- .../datasource/paimon/source/PaimonSplit.java | 23 + .../org/apache/doris/qe/SessionVariable.java | 18 + gensrc/thrift/PaloInternalService.thrift | 3 + gensrc/thrift/PlanNodes.thrift | 17 + .../paimon/test_paimon_blob_jni.out | 274 +++++++++++ .../paimon/test_paimon_blob_native.out | 274 +++++++++++ .../paimon/test_paimon_blob_jni.groovy | 443 ++++++++++++++++++ .../paimon/test_paimon_blob_native.groovy | 443 ++++++++++++++++++ 30 files changed, 3923 insertions(+), 15 deletions(-) create mode 100644 be/src/format_v2/blob/blob_file_reader.cpp create mode 100644 be/src/format_v2/blob/blob_file_reader.h create mode 100644 be/src/format_v2/blob/paimon_blob_column_reader.cpp create mode 100644 be/src/format_v2/blob/paimon_blob_column_reader.h create mode 100644 be/src/format_v2/blob/paimon_blob_materializer.cpp create mode 100644 be/src/format_v2/blob/paimon_blob_materializer.h create mode 100644 be/test/format_v2/blob/blob_file_reader_test.cpp create mode 100644 be/test/format_v2/blob/blob_test_util.h create mode 100644 be/test/format_v2/blob/paimon_blob_column_reader_test.cpp create mode 100644 be/test/format_v2/blob/paimon_blob_materializer_test.cpp create mode 100644 be/test/format_v2/blob/paimon_blob_test_util.h create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy diff --git a/be/src/core/data_type/data_type_file.h b/be/src/core/data_type/data_type_file.h index caa42fe866ecc6..def9b8a1b93e2e 100644 --- a/be/src/core/data_type/data_type_file.h +++ b/be/src/core/data_type/data_type_file.h @@ -31,6 +31,11 @@ class DataTypeFile final : public IDataType { public: static constexpr PrimitiveType PType = TYPE_FILE; static constexpr size_t FIELD_COUNT = 6; + static constexpr size_t URI_FIELD_INDEX = 0; + static constexpr size_t OFFSET_FIELD_INDEX = 1; + static constexpr size_t SIZE_FIELD_INDEX = 2; + static constexpr size_t CONTENT_TYPE_FIELD_INDEX = 3; + static constexpr size_t CHECKSUM_FIELD_INDEX = 4; static constexpr size_t INLINE_FIELD_INDEX = 5; DataTypeFile(); diff --git a/be/src/format_v2/blob/blob_file_reader.cpp b/be/src/format_v2/blob/blob_file_reader.cpp new file mode 100644 index 00000000000000..46f141710ccf61 --- /dev/null +++ b/be/src/format_v2/blob/blob_file_reader.cpp @@ -0,0 +1,244 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/blob/blob_file_reader.h" + +#include + +#include +#include +#include +#include + +#include "common/logging.h" +#include "core/pod_array.h" +#include "io/io_common.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris::format { +namespace { + +// Paimon managed .blob format, version 1. +// +// File layout (all fixed-width integers are little-endian): +// offset 0 EOF +// | | +// v v +// +---------------------------+--------------------+---------------+ +// | Non-NULL records | Row-length index | Footer | +// | R0 | R1 | ... | variable size | 5 bytes | +// +---------------------------+--------------------+---------------+ +// ^ | +// +-- index_size bytes-+ +// +// Each non-NULL record has 16 framing bytes around its payload: +// relative offset: 0 4 4 + N 12 + N 16 + N +// | | | | | +// v v v v v +// +---------+--------------------+---------------+---------+ +// | Magic | Payload | Record length | CRC32 | +// | 4 bytes | N bytes | 8 bytes | 4 bytes | +// +---------+--------------------+---------------+---------+ +// |<----------- CRC32 input -------------------->| +// |<-------------- record_length = N + 16 ---------------->| +// +// record_length is N + 16, including the magic, length and CRC fields. CRC32 covers +// the magic, payload and record_length, excluding the CRC field itself. +// +// Footer: +// file_size - 5 file_size - 1 file_size +// | | | +// v v v +// +-------------------------------------+---------------+ +// | index_size: uint32 (4 bytes) | version: 1 | +// | | (1 byte) | +// +-------------------------------------+---------------+ +// index_start = file_size - 5 - index_size +// +// The index has one entry per file row, including NULLs. Each entry is a zigzag varint +// encoding the delta from the previous row's record length (initial previous length: 0). +// A decoded length of -1 means NULL and consumes no record bytes. An empty non-NULL +// payload still has a 16-byte record, so it remains distinct from NULL. +// +// open() reads the last 5 bytes to locate the index, then reconstructs payload ranges +// by accumulating non-NULL record lengths from file offset 0. A payload starts 4 bytes +// after its record start and has record_length - 16 bytes. The expected row count comes +// from the caller's manifest metadata; it is not stored in the footer. These ranges +// allow descriptor output without reading payloads; read_payload() reads and validates +// an individual record only when inline bytes are requested. + +constexpr uint32_t BLOB_MAGIC = 1481511375; +constexpr int64_t BLOB_HEADER_SIZE = 4; +constexpr int64_t BLOB_TRAILER_SIZE = 12; // Record length (8) followed by CRC32 (4). +constexpr int64_t BLOB_RECORD_OVERHEAD = BLOB_HEADER_SIZE + BLOB_TRAILER_SIZE; +constexpr int64_t BLOB_FOOTER_SIZE = 5; // Index length (4) followed by format version (1). + +} // namespace + +Status BlobFileReader::_read_exact(int64_t offset, char* data, size_t size, + const io::IOContext* io_ctx) const { + if (offset < 0 || uint64_t(offset) > _reader->size() || + size > _reader->size() - uint64_t(offset)) { + return Status::Corruption("BLOB read exceeds file bounds"); + } + size_t read = 0; + RETURN_IF_ERROR(_reader->read_at(offset, Slice(data, size), &read, io_ctx)); + if (read != size) { + return Status::Corruption("Truncated BLOB file: expected {} bytes, got {}", size, read); + } + return Status::OK(); +} + +Status BlobFileReader::open(io::FileReaderSPtr reader, int64_t expected_rows, + const io::IOContext* io_ctx) { + _reader = std::move(reader); + _ranges.clear(); + if (_reader == nullptr || expected_rows < 0 || _reader->size() < BLOB_FOOTER_SIZE || + _reader->size() > uint64_t(std::numeric_limits::max())) { + return Status::Corruption("Invalid BLOB file size or row count"); + } + std::array footer; + // Read the 5-byte footer: index byte length (4 bytes) and format version (1 byte). + RETURN_IF_ERROR( + _read_exact(_reader->size() - footer.size(), footer.data(), footer.size(), io_ctx)); + if (footer[4] != 1) { + return Status::NotSupported("Unsupported Paimon BLOB version {}", + static_cast(footer[4])); + } + const uint64_t index_size = decode_fixed32_le(reinterpret_cast(footer.data())); + // Each row contributes one varint of 1..10 bytes. Check the upper bound by division + // so an invalid expected_rows cannot overflow a multiplication. + if (index_size > _reader->size() - footer.size() || index_size < uint64_t(expected_rows) || + (index_size + 9) / 10 > uint64_t(expected_rows)) { + return Status::Corruption("Invalid BLOB index length {} for {} rows", index_size, + expected_rows); + } + const int64_t index_offset = _reader->size() - footer.size() - index_size; + PODArray index; + index.resize(index_size); + // Read the delta/zigzag-varint record-length index to reconstruct each row's payload range. + RETURN_IF_ERROR(_read_exact(index_offset, index.data(), index.size(), io_ctx)); + _ranges.reserve(expected_rows); + + int64_t previous_length = 0; + int64_t offset = 0; + size_t cursor = 0; + // Each zigzag varint encodes the delta from the previous record length, including its + // 16 framing bytes. A length of -1 denotes NULL and occupies an index entry but no data. + while (cursor < index.size()) { + if (io_ctx != nullptr && io_ctx->should_stop) { + return Status::Cancelled("Paimon BLOB index read cancelled"); + } + if (_ranges.size() >= uint64_t(expected_rows)) { + return Status::Corruption("BLOB index has more rows than its manifest"); + } + uint64_t encoded = 0; + bool complete = false; + for (unsigned shift = 0; shift < 64; shift += 7) { + if (cursor == index.size()) { + return Status::Corruption("Truncated BLOB index varint"); + } + const auto byte = static_cast(index[cursor++]); + if (shift == 63 && byte > 1) { + return Status::Corruption("BLOB index varint overflow"); + } + encoded |= uint64_t(byte & 0x7f) << shift; + if ((byte & 0x80) == 0) { + complete = true; + break; + } + } + if (!complete) { + return Status::Corruption("BLOB index varint overflow"); + } + const auto delta = static_cast((encoded >> 1) ^ (0 - (encoded & 1))); + const __int128 length = __int128(previous_length) + delta; + if (length < -1 || length > std::numeric_limits::max() || + (length != -1 && (length < BLOB_RECORD_OVERHEAD || length > index_offset - offset))) { + return Status::Corruption("Invalid BLOB record length"); + } + previous_length = static_cast(length); + _ranges.push_back(length == -1 + ? ByteRange {.offset = -1, .size = 0} + : ByteRange {.offset = offset + BLOB_HEADER_SIZE, + .size = previous_length - BLOB_RECORD_OVERHEAD}); + if (length != -1) { + offset += previous_length; + } + } + // Every manifest row must have an index entry, and data records must end at the index. + if (_ranges.size() != uint64_t(expected_rows) || offset != index_offset) { + return Status::Corruption("BLOB index does not match manifest rows or data size"); + } + return Status::OK(); +} + +Status BlobFileReader::get_range(int64_t row, std::optional* range) const { + if (row < 0 || uint64_t(row) >= _ranges.size()) { + return Status::Corruption("BLOB row {} is outside file row count {}", row, _ranges.size()); + } + if (_ranges[row].offset < 0) { + range->reset(); + } else { + *range = _ranges[row]; + } + return Status::OK(); +} + +Status BlobFileReader::read_payload(const ByteRange& range, PODArray* bytes, + const io::IOContext* io_ctx) const { + // Non-NULL ranges from get_range() were already validated by open(). + DCHECK(_reader != nullptr); + DCHECK_GE(range.offset, BLOB_HEADER_SIZE); + DCHECK_GE(range.size, 0); + DCHECK_LE(uint64_t(range.offset), _reader->size()); + DCHECK_LE(uint64_t(range.size), _reader->size() - uint64_t(range.offset)); + DCHECK_GE(_reader->size() - uint64_t(range.offset) - uint64_t(range.size), + uint64_t(BLOB_TRAILER_SIZE)); + // Large values can still be described, so open() does not enforce the inline capacity. + if (uint64_t(range.size) > std::numeric_limits::max()) { + return Status::NotSupported("BLOB payload exceeds FILE inline VARBINARY capacity"); + } + // Framing and payload are contiguous. Read the whole record once so object storage does + // not need separate requests for the magic, payload and trailer. + const size_t record_size = static_cast(range.size) + BLOB_RECORD_OVERHEAD; + bytes->resize(record_size); + // Read this row's complete record: magic, payload, stored record length and CRC32. + RETURN_IF_ERROR( + _read_exact(range.offset - BLOB_HEADER_SIZE, bytes->data(), record_size, io_ctx)); + // If profiling shows record validation is costly, consider skipping CRC verification or + // reading only the indexed payload range. The latter also avoids reading framing bytes and + // the memmove below, but gives up both framing and CRC validation. Keep file/index bounds, + // exact-read checks and the caller's payload size limit even if these checks are relaxed. + const auto* record = reinterpret_cast(bytes->data()); + const auto* trailer = record + BLOB_HEADER_SIZE + range.size; + if (decode_fixed32_le(record) != BLOB_MAGIC || decode_fixed64_le(trailer) != record_size) { + return Status::Corruption("Invalid BLOB record framing"); + } + // CRC covers magic, payload and the stored record length, excluding the CRC itself. + const auto crc = crc32(0, record, static_cast(record_size - sizeof(uint32_t))); + if (uint32_t(crc) != decode_fixed32_le(trailer + sizeof(uint64_t))) { + return Status::Corruption("BLOB record checksum mismatch"); + } + // Strip framing in place and keep the buffer capacity for the next payload. + std::memmove(bytes->data(), bytes->data() + BLOB_HEADER_SIZE, static_cast(range.size)); + bytes->resize(static_cast(range.size)); + return Status::OK(); +} + +} // namespace doris::format diff --git a/be/src/format_v2/blob/blob_file_reader.h b/be/src/format_v2/blob/blob_file_reader.h new file mode 100644 index 00000000000000..7697b36aa6b3bb --- /dev/null +++ b/be/src/format_v2/blob/blob_file_reader.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "core/pod_array.h" +#include "io/fs/file_reader.h" + +namespace doris::format { + +// Physical Paimon managed-BLOB format, version 1. Row numbers are local to this file; +// table row ids, field ids, projection, and SQL FILE values belong to the caller. +class BlobFileReader { +public: + struct ByteRange { + // Absolute byte offset of the payload in this .blob file, excluding record framing. + int64_t offset; + // Payload bytes only; zero is a valid non-NULL empty value. + int64_t size; + }; + + // Read and decode the footer/index only. Data access requires a successful open(). + Status open(io::FileReaderSPtr reader, int64_t expected_rows, const io::IOContext* io_ctx); + // row is zero-based within this .blob file. SQL NULL returns nullopt, while an empty + // non-NULL value returns a ByteRange with size 0. Descriptor reads need only this index. + Status get_range(int64_t row, std::optional* range) const; + // Validate record framing and CRC, then copy the payload into the caller's tracked buffer. + // range must be an unmodified non-NULL range returned by get_range() after this reader's + // most recent successful open(). + // The buffer capacity is reused across calls; descriptor mode never calls this method. + Status read_payload(const ByteRange& range, PODArray* bytes, + const io::IOContext* io_ctx) const; + +private: + Status _read_exact(int64_t offset, char* data, size_t size, const io::IOContext* io_ctx) const; + + io::FileReaderSPtr _reader; + // Decoded index retained for this file, using tracked memory like the temporary buffers. + // Entries follow file row order; a negative offset represents SQL NULL. + PODArray _ranges; +}; + +} // namespace doris::format diff --git a/be/src/format_v2/blob/paimon_blob_column_reader.cpp b/be/src/format_v2/blob/paimon_blob_column_reader.cpp new file mode 100644 index 00000000000000..5fdc9a1847f357 --- /dev/null +++ b/be/src/format_v2/blob/paimon_blob_column_reader.cpp @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/blob/paimon_blob_column_reader.h" + +#include +#include +#include + +#include "core/data_type/data_type_file.h" +#include "core/pod_array.h" +#include "format_v2/blob/blob_file_reader.h" +#include "io/file_factory.h" +#include "io/io_common.h" + +namespace doris::format::paimon { + +Status PaimonBlobColumnReader::validate_blob_files(std::vector* files, + int64_t anchor_first_row_id, + int64_t anchor_row_count) { + if (files->empty()) { + return Status::InvalidArgument("Cannot validate an empty managed BLOB file group"); + } + for (const auto* file : *files) { + if (!file->__isset.path || !file->__isset.file_size || !file->__isset.first_row_id || + !file->__isset.row_count || file->path.empty() || file->file_size < 5 || + file->row_count <= 0 || file->first_row_id < anchor_first_row_id || + file->first_row_id > anchor_first_row_id + anchor_row_count || + file->row_count > anchor_first_row_id + anchor_row_count - file->first_row_id) { + return Status::InvalidArgument("Invalid managed BLOB file descriptor"); + } + } + std::ranges::sort(*files, {}, [](const auto* file) { return file->first_row_id; }); + int64_t next = anchor_first_row_id; + for (const auto* file : *files) { + if (file->first_row_id != next) { + return Status::Corruption("BLOB column {} has overlapping or missing row ranges", + file->field_id); + } + next += file->row_count; + } + if (next != anchor_first_row_id + anchor_row_count) { + return Status::Corruption("BLOB column {} does not cover its Parquet anchor", + files->front()->field_id); + } + return Status::OK(); +} + +PaimonBlobColumnReader::PaimonBlobColumnReader(const ColumnDefinition& column, + const std::vector& files, + bool blob_as_descriptor) + : _blob_column_name(column.name), + _blob_column_type(column.type), + _blob_as_descriptor(blob_as_descriptor) { + _blob_files.reserve(files.size()); + for (const auto* file : files) { + _blob_files.push_back(BlobFileState {.blob_metadata = *file, .blob_reader = nullptr}); + } +} + +PaimonBlobColumnReader::~PaimonBlobColumnReader() = default; + +Status PaimonBlobColumnReader::_open_blob_file(BlobFileState* blob_file, + const ReadContext& context) { + if (blob_file->blob_reader) { + return Status::OK(); + } + io::FileDescription description; + description.path = blob_file->blob_metadata.path; + description.file_size = blob_file->blob_metadata.file_size; + description.is_immutable = true; + description.fs_name = context.fs_name; + description.file_cache_admission = context.file_cache_admission; + auto reader = FileFactory::create_file_reader( + context.system_properties, description, + FileFactory::get_reader_options(context.runtime_state, description)); + if (!reader.has_value()) { + return std::move(reader).error(); + } + if (reader.value()->size() != uint64_t(description.file_size)) { + return Status::Corruption("BLOB file size differs from its manifest: {}", description.path); + } + auto blob = std::make_unique(); + RETURN_IF_ERROR(blob->open(std::move(reader).value(), blob_file->blob_metadata.row_count, + context.io_ctx)); + blob_file->blob_reader = std::move(blob); + return Status::OK(); +} + +Status PaimonBlobColumnReader::read_blob(const ColumnInt64& anchor_row_positions, + int64_t anchor_first_row_id, const ReadContext& context, + MutableColumnPtr* blob_column_result) { + auto blob_column = _blob_column_type->create_column(); + blob_column->reserve(anchor_row_positions.size()); + if (_blob_files.empty()) { + // Schema evolution can leave this field absent for the whole anchor. A partial + // range is different: validate_blob_files() rejects holes in an existing file group. + if (!_blob_column_type->is_nullable()) { + return Status::Corruption("Missing required BLOB column {}", _blob_column_name); + } + blob_column->insert_many_defaults(anchor_row_positions.size()); + *blob_column_result = std::move(blob_column); + return Status::OK(); + } + // Reuse a tracked buffer within this batch; release its capacity when the batch finishes. + PODArray payload; + // The output mode is fixed for this reader, so unused FILE children remain NULL. + auto blob_file_value = Field::create_field(File(DataTypeFile::FIELD_COUNT)); + auto& blob_file_fields = blob_file_value.get(); + BlobFileState* blob_file = nullptr; + for (const auto anchor_row_position : anchor_row_positions.get_data()) { + if (context.io_ctx && context.io_ctx->should_stop) { + return Status::Cancelled("Paimon BLOB read cancelled"); + } + // Convert the original Parquet file position to a Paimon logical row ID. + // Example: anchor origin 1000 + file position 4 = logical row 1004. + const int64_t paimon_row_id = anchor_first_row_id + anchor_row_position; + // Consecutive selected rows usually share a source. Only search and open when the + // row leaves its range; retain correctness for non-monotonic row positions too. + if (blob_file == nullptr || paimon_row_id < blob_file->blob_metadata.first_row_id || + paimon_row_id - blob_file->blob_metadata.first_row_id >= + blob_file->blob_metadata.row_count) { + // Validated ranges are contiguous. upper_bound finds the next file's start; + // its predecessor covers paimon_row_id, including an exact file-start match. + auto it = std::ranges::upper_bound(_blob_files, paimon_row_id, {}, + [](const BlobFileState& blob_file) { + return blob_file.blob_metadata.first_row_id; + }); + if (it == _blob_files.begin()) { + return Status::Corruption("BLOB column has no file for row {}", paimon_row_id); + } + --it; + blob_file = &*it; + RETURN_IF_ERROR(_open_blob_file(blob_file, context)); + } + std::optional payload_range; + // Rebase against this .blob file's origin, which can differ from the anchor's. + // For a .blob starting at 1003, logical row 1004 is file-local record 1. + const int64_t blob_row_position = paimon_row_id - blob_file->blob_metadata.first_row_id; + RETURN_IF_ERROR(blob_file->blob_reader->get_range(blob_row_position, &payload_range)); + if (!payload_range) { + if (!_blob_column_type->is_nullable()) { + return Status::Corruption("NULL in required BLOB column {}", _blob_column_name); + } + blob_column->insert_default(); + continue; + } + if (_blob_as_descriptor) { + blob_file_fields[DataTypeFile::URI_FIELD_INDEX] = + Field::create_field(blob_file->blob_metadata.path); + blob_file_fields[DataTypeFile::OFFSET_FIELD_INDEX] = + Field::create_field(payload_range->offset); + blob_file_fields[DataTypeFile::SIZE_FIELD_INDEX] = + Field::create_field(payload_range->size); + } else { + // The index exposes the payload length before any payload buffer allocation or I/O. + if (payload_range->size > context.blob_max_size) { + return Status::InvalidArgument( + "Paimon BLOB column '{}' inline size {} bytes exceeds " + "paimon_blob_max_size={} bytes. Increase paimon_blob_max_size or use " + "blob-as-descriptor=true", + _blob_column_name, payload_range->size, context.blob_max_size); + } + // Inline output contains only the payload; metadata children remain NULL. + RETURN_IF_ERROR( + blob_file->blob_reader->read_payload(*payload_range, &payload, context.io_ctx)); + blob_file_fields[DataTypeFile::INLINE_FIELD_INDEX] = + Field::create_field( + StringView(payload.data(), static_cast(payload.size()))); + } + // Large StringView values borrow payload's buffer; insert copies the bytes into + // the FILE column before the next read can resize or overwrite that buffer. + blob_column->insert(blob_file_value); + } + *blob_column_result = std::move(blob_column); + return Status::OK(); +} + +} // namespace doris::format::paimon diff --git a/be/src/format_v2/blob/paimon_blob_column_reader.h b/be/src/format_v2/blob/paimon_blob_column_reader.h new file mode 100644 index 00000000000000..af2fcdd4ff61d5 --- /dev/null +++ b/be/src/format_v2/blob/paimon_blob_column_reader.h @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "core/column/column_vector.h" +#include "format_v2/column_data.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { +class RuntimeState; +namespace io { +struct FileSystemProperties; +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format { +class BlobFileReader; +} // namespace doris::format + +namespace doris::format::paimon { + +// Reads one top-level BLOB column using Paimon Default blob storage. Managed .blob files +// are joined by logical row ID; descriptor-only/external-storage layouts are handled by JNI. +// Output may contain inline bytes or a descriptor, independently of this storage restriction. +// This reader has no knowledge of output Block positions. +class PaimonBlobColumnReader { +public: + // Borrowed only for read_blob(); no owning TableReader state is retained. + struct ReadContext { + const io::FileSystemProperties& system_properties; + RuntimeState* runtime_state; + const io::IOContext* io_ctx; + const std::string& fs_name; + bool file_cache_admission; + // Per-value inline payload limit; descriptor output does not read payload bytes. + int64_t blob_max_size; + }; + + // Sort a nonempty group for one field and verify contiguous coverage of the whole anchor. + // The caller validates field IDs and the non-overflowing anchor range first. This is also + // used for unprojected groups, so corrupt metadata cannot be mistaken for a missing column. + static Status validate_blob_files(std::vector* files, + int64_t anchor_first_row_id, int64_t anchor_row_count); + + // Copies descriptors from an already validated/sorted group; borrows no metadata pointers. + // An empty group means this field was absent when the anchor was written. + PaimonBlobColumnReader(const ColumnDefinition& column, + const std::vector& files, + bool blob_as_descriptor); + + ~PaimonBlobColumnReader(); + + // anchor_row_positions are zero-based original positions in the complete Parquet file, + // preserved across filtering and byte splits. Returns one FILE value per input position; + // files and decoded indexes are opened lazily. + Status read_blob(const ColumnInt64& anchor_row_positions, int64_t anchor_first_row_id, + const ReadContext& context, MutableColumnPtr* blob_column_result); + +private: + struct BlobFileState { + TPaimonBlobFileDesc blob_metadata; + // Created on first access; metadata and decoded indexes are reused across batches. + std::unique_ptr blob_reader; + }; + + Status _open_blob_file(BlobFileState* blob_file, const ReadContext& context); + + std::string _blob_column_name; + DataTypePtr _blob_column_type; + std::vector _blob_files; + bool _blob_as_descriptor; +}; + +} // namespace doris::format::paimon diff --git a/be/src/format_v2/blob/paimon_blob_materializer.cpp b/be/src/format_v2/blob/paimon_blob_materializer.cpp new file mode 100644 index 00000000000000..544c52ebfd454e --- /dev/null +++ b/be/src/format_v2/blob/paimon_blob_materializer.cpp @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/blob/paimon_blob_materializer.h" + +#include +#include + +#include "core/block/block.h" +#include "core/data_type/data_type_nullable.h" +#include "format_v2/blob/paimon_blob_column_reader.h" + +namespace doris::format::paimon { + +Status PaimonBlobMaterializer::init(const std::map& options) { + _blob_as_descriptor = false; + const auto it = options.find("blob-as-descriptor"); + if (it != options.end()) { + if (it->second != "true" && it->second != "false") { + return Status::InvalidArgument("Invalid blob-as-descriptor value {}", it->second); + } + _blob_as_descriptor = it->second == "true"; + } + return Status::OK(); +} + +void PaimonBlobMaterializer::reset() { + _blob_column_readers.clear(); + _blob_column_names.clear(); + _anchor_first_row_id = -1; + _anchor_row_count = 0; +} + +Status PaimonBlobMaterializer::prepare_split( + const TPaimonFileDesc& params, const std::vector& projected_columns) { + reset(); + if (!params.__isset.blob_files) { + return Status::OK(); + } + if (!params.__isset.anchor_first_row_id || !params.__isset.anchor_row_count || + params.anchor_first_row_id < 0 || params.anchor_row_count <= 0 || + params.anchor_first_row_id > + std::numeric_limits::max() - params.anchor_row_count) { + return Status::InvalidArgument("Missing or invalid native Paimon BLOB split metadata"); + } + _anchor_first_row_id = params.anchor_first_row_id; + _anchor_row_count = params.anchor_row_count; + std::map> files_by_field; + for (const auto& file : params.blob_files) { + if (!file.__isset.field_id || file.field_id < 0) { + return Status::InvalidArgument("Invalid managed BLOB field id"); + } + files_by_field[file.field_id].push_back(&file); + } + for (auto& entry : files_by_field) { + RETURN_IF_ERROR(PaimonBlobColumnReader::validate_blob_files( + &entry.second, _anchor_first_row_id, _anchor_row_count)); + } + // Resolve table output positions and field ids once per split. Only projected FILE columns + // retain readers; unprojected BLOB files never open readers or keep decoded indexes alive. + const std::vector empty_blob_files; + for (size_t index = 0; index < projected_columns.size(); ++index) { + const auto& column = projected_columns[index]; + if (remove_nullable(column.type)->get_primitive_type() != TYPE_FILE) { + continue; + } + if (!column.has_identifier_field_id()) { + return Status::InvalidArgument("Native Paimon BLOB requires a field id"); + } + const auto it = files_by_field.find(column.get_identifier_field_id()); + auto reader = std::make_unique( + column, it == files_by_field.end() ? empty_blob_files : it->second, + _blob_as_descriptor); + _blob_column_names.insert(column.name); + _blob_column_readers.emplace(GlobalIndex(index), std::move(reader)); + } + return Status::OK(); +} + +Status PaimonBlobMaterializer::materialize(const ColumnInt64& anchor_row_positions, + Block* table_block, const ReadContext& context) { + if (!has_blob_columns() || table_block->rows() == 0) { + return Status::OK(); + } + if (anchor_row_positions.size() != table_block->rows()) { + return Status::InternalError("BLOB row positions and Parquet output are not aligned"); + } + for (const auto anchor_row_position : anchor_row_positions.get_data()) { + if (anchor_row_position < 0 || anchor_row_position >= _anchor_row_count) { + return Status::Corruption("Parquet row position is outside its Paimon row range"); + } + } + for (auto& [output_index, reader] : _blob_column_readers) { + MutableColumnPtr blob_column_result; + RETURN_IF_ERROR(reader->read_blob(anchor_row_positions, _anchor_first_row_id, context, + &blob_column_result)); + table_block->replace_by_position(output_index.value(), std::move(blob_column_result)); + } + return Status::OK(); +} + +} // namespace doris::format::paimon diff --git a/be/src/format_v2/blob/paimon_blob_materializer.h b/be/src/format_v2/blob/paimon_blob_materializer.h new file mode 100644 index 00000000000000..a44504ef783d2f --- /dev/null +++ b/be/src/format_v2/blob/paimon_blob_materializer.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "core/column/column_vector.h" +#include "format_v2/blob/paimon_blob_column_reader.h" +#include "format_v2/column_data.h" +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { +class Block; +} // namespace doris + +namespace doris::format::paimon { + +// Coordinates Default blob storage materialization across projected columns. Column readers +// own file selection and FILE construction; this class owns anchor row validation and output +// Block positions. PaimonReader supplies original file row positions after Parquet filtering. +class PaimonBlobMaterializer { +public: + using ReadContext = PaimonBlobColumnReader::ReadContext; + + Status init(const std::map& options); + // Clears the previous split, validates row ranges and resolves projected FILE columns. + // This does not open BLOB files. An absent blob_files field disables materialization; + // a present but empty list still creates readers that fill missing nullable columns with NULL. + Status prepare_split(const TPaimonFileDesc& params, + const std::vector& projected_columns); + Status materialize(const ColumnInt64& anchor_row_positions, Block* table_block, + const ReadContext& context); + // Releases per-split files and indexes while retaining query options. + void reset(); + bool has_blob_columns() const { return !_blob_column_readers.empty(); } + const std::set& blob_column_names() const { return _blob_column_names; } + +private: + bool _blob_as_descriptor = false; + // Logical range of the complete Parquet anchor. All byte splits use the same origin; + // ROW_POSITION remains relative to the whole file, including rows skipped by filtering. + int64_t _anchor_first_row_id = -1; + int64_t _anchor_row_count = 0; + std::map> _blob_column_readers; + std::set _blob_column_names; +}; + +} // namespace doris::format::paimon diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index b5f7e7c89bf367..802888c9b7ba80 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -197,6 +197,8 @@ std::string virtual_column_type_to_string(TableVirtualColumnType type) { return "LAST_UPDATED_SEQUENCE_NUMBER"; case TableVirtualColumnType::ICEBERG_ROWID: return "ICEBERG_ROWID"; + case TableVirtualColumnType::PAIMON_BLOB: + return "PAIMON_BLOB"; } return "UNKNOWN"; } @@ -2270,6 +2272,13 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab mapping->table_column_name = table_column.name; mapping->table_type = table_column.type; mapping->variant_access_paths = table_column.variant_access_paths; + if (_options.paimon_blob_columns.contains(table_column.name)) { + // Managed BLOBs live outside the Parquet anchor. Wait for PaimonBlobMaterializer + // before evaluating their predicates; do not synthesize missing-column constants. + mapping->virtual_column_type = TableVirtualColumnType::PAIMON_BLOB; + mapping->filter_conversion = FilterConversionType::FINALIZE_ONLY; + return Status::OK(); + } // Row-lineage names are Iceberg metadata contracts, not reserved names in generic Hive, // Hudi, or Paimon schemas. Only the Iceberg reader may opt into virtual synthesis. const auto row_lineage_type = diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index bc08c0049b2132..74535f14e4f7f6 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -63,6 +64,9 @@ enum TableVirtualColumnType { // Doris internal Iceberg row locator column `__DORIS_ICEBERG_ROWID_COL__`. // It is a struct used by delete/update/merge, not the Iceberg `_row_id`. ICEBERG_ROWID = 3, + // Paimon Default blob storage: FILE values materialized from managed .blob files + // after physical columns and before table predicates. + PAIMON_BLOB = 4, }; enum class FilterConversionType { @@ -164,6 +168,9 @@ struct TableColumnMapperOptions { bool reject_missing_required_field = false; bool allow_idless_complex_wrapper_projection = false; bool enable_row_lineage_virtual_columns = false; + // Projected Paimon managed BLOB columns materialized as FILE by PaimonBlobMaterializer. + // They have no source in the anchor and must bypass constants and file predicate pushdown. + std::set paimon_blob_columns {}; std::string debug_string() const; }; diff --git a/be/src/format_v2/jni/paimon_jni_reader.cpp b/be/src/format_v2/jni/paimon_jni_reader.cpp index 1ec6b7c9d3b7a9..fd53b0b440d185 100644 --- a/be/src/format_v2/jni/paimon_jni_reader.cpp +++ b/be/src/format_v2/jni/paimon_jni_reader.cpp @@ -88,6 +88,7 @@ std::string PaimonJniReader::connector_class() const { Status PaimonJniReader::build_scanner_params(std::map* params) const { DORIS_CHECK(params != nullptr); DORIS_CHECK(_scan_params != nullptr); + DORIS_CHECK(_runtime_state != nullptr); params->clear(); const auto& paimon_params = _current_range.table_format_params.paimon_params; @@ -102,6 +103,8 @@ Status PaimonJniReader::build_scanner_params(std::map* !_scan_params->serialized_table_cache_key.empty() ? _scan_params->serialized_table_cache_key : generate_uuid_string(); + (*params)["paimon_blob_max_size"] = + std::to_string(_runtime_state->query_options().paimon_blob_max_size); if (_scan_params->__isset.paimon_options && !_scan_params->paimon_options.empty()) { for (const auto& kv : _scan_params->paimon_options) { @@ -120,7 +123,6 @@ Status PaimonJniReader::build_scanner_params(std::map* auto enable_io_manager_it = params->find(enable_io_manager_key); if (enable_io_manager_it != params->end() && iequal(enable_io_manager_it->second, "true") && params->find(io_manager_tmp_dir_key) == params->end()) { - DORIS_CHECK(_runtime_state != nullptr); // Keep Format V2 consistent with the legacy JNI path. Paimon creates and // removes its own paimon-* child directory under these Doris storage-root scoped // parent directories. diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 3c1c098af93610..ec91f925b8485a 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -1082,6 +1082,7 @@ Status IcebergTableReader::materialize_virtual_columns(Block* table_block) { RETURN_IF_ERROR(_materialize_iceberg_rowid(table_block, column_idx)); break; case format::TableVirtualColumnType::INVALID: + case format::TableVirtualColumnType::PAIMON_BLOB: break; } } diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 2ec6bde53f2a00..d90560bc6493c6 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -20,10 +20,12 @@ #include #include +#include #include #include #include +#include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_map.h" @@ -33,11 +35,13 @@ #include "exprs/vexpr_context.h" #include "format/table/deletion_vector_reader.h" #include "format/table/paimon_reader.h" +#include "format_v2/blob/paimon_blob_materializer.h" #include "format_v2/column_mapper.h" #include "format_v2/jni/paimon_jni_reader.h" #include "format_v2/table/paimon_rust_table_reader.h" #include "format_v2/table/schema_history_util.h" #include "gen_cpp/PlanNodes_types.h" +#include "runtime/runtime_state.h" namespace doris::format::paimon { namespace { @@ -235,16 +239,41 @@ Status apply_paimon_timestamp_semantics(format::ColumnDefinition* column) { } // namespace +PaimonReader::PaimonReader() : _blob_materializer(std::make_unique()) {} + +PaimonReader::~PaimonReader() = default; + +Status PaimonReader::init(format::TableReadOptions&& options) { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + const std::map empty_options; + return _blob_materializer->init(_scan_params && _scan_params->__isset.paimon_options + ? _scan_params->paimon_options + : empty_options); +} + +Status PaimonReader::abort_split() { + auto status = TableReader::abort_split(); + _blob_materializer->reset(); + _blob_row_position_index.reset(); + return status; +} + +Status PaimonReader::close() { + auto status = TableReader::close(); + _blob_materializer->reset(); + _blob_row_position_index.reset(); + return status; +} + Status PaimonReader::prepare_split(const format::SplitReadOptions& options) { { - // Derived schema selection is additive to, not nested around, the common base timers. + // Add derived preparation time without nesting the base class's counters. SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.prepare_split_timer); - _split_schema_id = -1; const auto& paimon_params = options.current_range.table_format_params.paimon_params; - if (paimon_params.__isset.schema_id) { - _split_schema_id = paimon_params.schema_id; - } + _split_schema_id = paimon_params.__isset.schema_id ? paimon_params.schema_id : -1; + _blob_row_position_index.reset(); + RETURN_IF_ERROR(_blob_materializer->prepare_split(paimon_params, _projected_columns)); } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); SCOPED_TIMER(_profile.total_timer); @@ -292,9 +321,38 @@ Status PaimonReader::customize_file_scan_request(format::FileScanRequest* file_r DORIS_CHECK(file_request != nullptr); RETURN_IF_ERROR(format::TableReader::customize_file_scan_request(file_request)); file_request->variant_schema_overrides = _variant_schema_overrides; + if (_blob_materializer->has_blob_columns()) { + // Request original file positions alongside surviving rows for the BLOB row-ID join. + const auto id = format::LocalColumnId(format::ROW_POSITION_COLUMN_ID); + _append_file_scan_column(file_request, id, &file_request->non_predicate_columns); + _blob_row_position_index = file_request->local_positions.at(id); + } return Status::OK(); } +void PaimonReader::configure_mapper_options(format::TableColumnMapperOptions* options) const { + options->paimon_blob_columns = _blob_materializer->blob_column_names(); +} + +Status PaimonReader::materialize_virtual_columns(Block* table_block) { + if (!_blob_row_position_index || table_block->rows() == 0) { + return Status::OK(); + } + const auto& anchor_row_positions = assert_cast( + *_data_reader.block_template.get_by_position(_blob_row_position_index->value()).column); + return _blob_materializer->materialize( + anchor_row_positions, table_block, + {.system_properties = *_system_properties, + .runtime_state = _runtime_state, + .io_ctx = _io_ctx.get(), + .fs_name = _current_file_range_desc.fs_name, + .file_cache_admission = !_current_file_range_desc.__isset.file_cache_admission || + _current_file_range_desc.file_cache_admission, + .blob_max_size = _runtime_state != nullptr + ? _runtime_state->query_options().paimon_blob_max_size + : TQueryOptions().paimon_blob_max_size}); +} + Status PaimonReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc, bool* has_delete_file) { DORIS_CHECK(desc != nullptr); diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index f0ac448828b3a6..ac08771404e7a9 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include #include #include "format_v2/table_reader.h" @@ -27,11 +29,17 @@ struct DeleteFileDesc; } namespace doris::format::paimon { +class PaimonBlobMaterializer; + class PaimonReader final : public format::TableReader { public: ENABLE_FACTORY_CREATOR(PaimonReader); - ~PaimonReader() final = default; + PaimonReader(); + ~PaimonReader() final; + Status init(format::TableReadOptions&& options) override; Status prepare_split(const format::SplitReadOptions& options) override; + Status abort_split() override; + Status close() override; #ifdef BE_TEST void TEST_set_scan_params(TFileScanRangeParams* params) { _scan_params = params; } @@ -56,6 +64,8 @@ class PaimonReader final : public format::TableReader { format::TableColumnMappingMode mapping_mode() const override; Status annotate_file_schema(std::vector* file_schema) override; Status customize_file_scan_request(format::FileScanRequest* file_request) override; + void configure_mapper_options(format::TableColumnMapperOptions* options) const override; + Status materialize_virtual_columns(Block* table_block) override; Status _parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc, bool* has_delete_file) override; @@ -63,6 +73,9 @@ class PaimonReader final : public format::TableReader { private: int64_t _split_schema_id = -1; std::vector _variant_schema_overrides; + std::unique_ptr _blob_materializer; + // File-local Block slot holding ROW_POSITION values, not a Paimon logical row ID. + std::optional _blob_row_position_index; }; // Paimon scans can contain both native data-file splits and serialized JNI splits in the same diff --git a/be/test/format_v2/blob/blob_file_reader_test.cpp b/be/test/format_v2/blob/blob_file_reader_test.cpp new file mode 100644 index 00000000000000..50f4df334e5f13 --- /dev/null +++ b/be/test/format_v2/blob/blob_file_reader_test.cpp @@ -0,0 +1,356 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/blob/blob_file_reader.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "format_v2/blob/blob_test_util.h" +#include "io/io_common.h" + +namespace doris::format { +namespace { + +using blob_test::encode_record_lengths; +using blob_test::finish_blob_file; +using blob_test::make_blob_file; + +class MemoryBlobFile final : public io::FileReader { +public: + // A nonzero offset represents a sparse file: only its index/footer are backed by memory. + explicit MemoryBlobFile(std::string bytes, size_t content_offset = 0) + : _bytes(std::move(bytes)), _content_offset(content_offset) {} + + Status close() override { + _closed = true; + return Status::OK(); + } + const io::Path& path() const override { return _path; } + size_t size() const override { return _content_offset + _bytes.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 0; } + + std::vector> read_requests; + std::optional short_read_offset; + std::optional error_offset; + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext*) override { + read_requests.emplace_back(offset, result.size); + if (error_offset == offset || offset < _content_offset) { + return Status::IOError("Injected BLOB read failure"); + } + *bytes_read = std::min(result.size, size() - offset); + if (short_read_offset == offset && *bytes_read > 0) { + --*bytes_read; + } + if (*bytes_read > 0) { + std::memcpy(result.data, _bytes.data() + offset - _content_offset, *bytes_read); + } + return Status::OK(); + } + +private: + io::Path _path {"memory.blob"}; + std::string _bytes; + size_t _content_offset; + bool _closed = false; +}; + +TEST(BlobFileReaderTest, ReadsPaimon142WriterBytes) { + // Generated with Paimon 1.4.2 BlobFormatWriter for ["hello", NULL, empty, 00FF]. + // Fixed bytes independently pin little-endian framing, CRC32 and NULL/empty index deltas. + const std::string bytes = std::string( + "\xcf\x11\x4e\x58hello\x15\x00\x00\x00\x00\x00\x00\x00" + "\xc8\xf6\xa7\x51\xcf\x11\x4e\x58\x10\x00\x00\x00" + "\x00\x00\x00\x00\x53\x7f\xdf\x03\xcf\x11\x4e\x58" + "\x00\xff\x12\x00\x00\x00\x00\x00\x00\x00\xef\xc8" + "\x8a\x40\x2a\x2b\x22\x04\x04\x00\x00\x00\x01", + 64); + EXPECT_EQ(bytes, make_blob_file({"hello", std::nullopt, "", std::string("\0\xff", 2)})); + auto file = std::make_shared(bytes); + BlobFileReader reader; + ASSERT_TRUE(reader.open(file, 4, nullptr).ok()); + + // Descriptor access reads only the footer and index, never record bytes. + ASSERT_EQ(file->read_requests.size(), 2); + EXPECT_EQ(file->read_requests[0], std::make_pair(size_t {59}, size_t {5})); + EXPECT_EQ(file->read_requests[1], std::make_pair(size_t {55}, size_t {4})); + std::optional range; + ASSERT_TRUE(reader.get_range(0, &range).ok()); + ASSERT_TRUE(range.has_value()); + EXPECT_EQ(range->offset, 4); + EXPECT_EQ(range->size, 5); + PODArray payload; + ASSERT_TRUE(reader.read_payload(*range, &payload, nullptr).ok()); + EXPECT_EQ(std::string(payload.data(), payload.size()), "hello"); + + ASSERT_TRUE(reader.get_range(1, &range).ok()); + EXPECT_FALSE(range.has_value()); + ASSERT_TRUE(reader.get_range(2, &range).ok()); + ASSERT_TRUE(range.has_value()); + EXPECT_EQ(range->offset, 25); + EXPECT_EQ(range->size, 0); + ASSERT_TRUE(reader.read_payload(*range, &payload, nullptr).ok()); + EXPECT_TRUE(payload.empty()); + ASSERT_TRUE(reader.get_range(3, &range).ok()); + ASSERT_TRUE(range.has_value()); + EXPECT_EQ(range->offset, 41); + EXPECT_EQ(range->size, 2); + ASSERT_TRUE(reader.read_payload(*range, &payload, nullptr).ok()); + EXPECT_EQ(std::string(payload.data(), payload.size()), std::string("\0\xff", 2)); +} + +TEST(BlobFileReaderTest, BinaryPayloadsAndVarintLengthTransitions) { + std::vector> values {std::nullopt, ""}; + for (const size_t size : {47, 48, 49, 111, 112, 113, 127, 128, 129, 8175, 8176, 8177}) { + values.emplace_back(std::string(size, 'x')); + } + values.emplace_back(std::nullopt); + values.emplace_back(""); + values.emplace_back("short"); + std::string all_bytes; + for (int byte = 0; byte < 256; ++byte) { + all_bytes.push_back(static_cast(byte)); + } + values.emplace_back(all_bytes); + values.emplace_back("你好Doris🌍"); + BlobFileReader reader; + ASSERT_TRUE(reader.open(std::make_shared(make_blob_file(values)), + static_cast(values.size()), nullptr) + .ok()); + PODArray payload; + int64_t record_offset = 0; + for (size_t row = 0; row < values.size(); ++row) { + SCOPED_TRACE(row); + std::optional range; + ASSERT_TRUE(reader.get_range(static_cast(row), &range).ok()); + if (!values[row]) { + EXPECT_FALSE(range.has_value()); + continue; + } + ASSERT_TRUE(range.has_value()); + EXPECT_EQ(range->offset, record_offset + 4); + EXPECT_EQ(range->size, values[row]->size()); + ASSERT_TRUE(reader.read_payload(*range, &payload, nullptr).ok()); + EXPECT_EQ(std::string(payload.data(), payload.size()), *values[row]); + record_offset += static_cast(values[row]->size()) + 16; + } +} + +TEST(BlobFileReaderTest, EmptyAndAllNullFiles) { + for (const size_t row_count : {0, 4}) { + SCOPED_TRACE(row_count); + auto file = std::make_shared( + make_blob_file(std::vector>(row_count))); + BlobFileReader reader; + ASSERT_TRUE(reader.open(file, static_cast(row_count), nullptr).ok()); + for (size_t row = 0; row < row_count; ++row) { + std::optional range; + ASSERT_TRUE(reader.get_range(static_cast(row), &range).ok()); + EXPECT_FALSE(range.has_value()); + } + EXPECT_EQ(file->read_requests.size(), 2); + } +} + +TEST(BlobFileReaderTest, ReopeningClearsPreviousIndex) { + BlobFileReader reader; + ASSERT_TRUE(reader.open(std::make_shared(make_blob_file({"old", "", "end"})), 3, + nullptr) + .ok()); + ASSERT_TRUE(reader.open(std::make_shared(make_blob_file({"new"})), 1, nullptr) + .ok()); + std::optional range; + EXPECT_TRUE(reader.get_range(1, &range).is()); + ASSERT_TRUE(reader.get_range(0, &range).ok()); + ASSERT_TRUE(range.has_value()); + PODArray payload; + ASSERT_TRUE(reader.read_payload(*range, &payload, nullptr).ok()); + EXPECT_EQ(std::string(payload.data(), payload.size()), "new"); +} + +TEST(BlobFileReaderTest, RejectsOutOfBoundsRows) { + BlobFileReader reader; + ASSERT_TRUE( + reader.open(std::make_shared(make_blob_file({"a"})), 1, nullptr).ok()); + for (const int64_t row : {int64_t {-1}, int64_t {1}, std::numeric_limits::max()}) { + std::optional range; + EXPECT_TRUE(reader.get_range(row, &range).is()); + } +} + +TEST(BlobFileReaderTest, RejectsInvalidFileSizeAndRowCount) { + BlobFileReader reader; + EXPECT_TRUE(reader.open(nullptr, 1, nullptr).is()); + EXPECT_TRUE(reader.open(std::make_shared("1234"), 0, nullptr) + .is()); + EXPECT_TRUE(reader.open(std::make_shared(make_blob_file({"a"})), -1, nullptr) + .is()); +} + +TEST(BlobFileReaderTest, RejectsUnknownVersion) { + std::string bytes = make_blob_file({"a"}); + bytes.back() = '\2'; + BlobFileReader reader; + const auto status = reader.open(std::make_shared(bytes), 1, nullptr); + EXPECT_TRUE(status.is()) << status; +} + +TEST(BlobFileReaderTest, RejectsInvalidIndexLength) { + for (const uint32_t index_size : {0U, 100U, std::numeric_limits::max()}) { + SCOPED_TRACE(index_size); + std::string bytes = make_blob_file({"a"}); + bytes.resize(bytes.size() - 5); + blob_test::append_little_endian(&bytes, index_size, 4); + bytes.push_back('\1'); + BlobFileReader reader; + EXPECT_TRUE(reader.open(std::make_shared(bytes), 1, nullptr) + .is()); + } +} + +TEST(BlobFileReaderTest, RejectsTruncatedAndOverflowingVarints) { + const std::vector indexes {std::string(1, '\x80'), std::string(9, '\x80') + '\x02', + std::string(10, '\x80')}; + for (const auto& index : indexes) { + BlobFileReader reader; + const auto status = reader.open( + std::make_shared(finish_blob_file("", index)), 1, nullptr); + EXPECT_TRUE(status.is()) << status; + } +} + +TEST(BlobFileReaderTest, RejectsInvalidRecordLengths) { + for (const int64_t length : {-2, 0, 15, 17}) { + SCOPED_TRACE(length); + BlobFileReader reader; + const auto status = + reader.open(std::make_shared(finish_blob_file( + std::string(16, '\0'), encode_record_lengths({length}))), + 1, nullptr); + EXPECT_TRUE(status.is()) << status; + } + // The second delta would overflow the signed record length when added to the first. + const std::string index = encode_record_lengths({16}) + + encode_record_lengths({std::numeric_limits::max()}); + BlobFileReader reader; + EXPECT_TRUE(reader.open(std::make_shared( + finish_blob_file(std::string(16, '\0'), index)), + 2, nullptr) + .is()); +} + +TEST(BlobFileReaderTest, RejectsManifestRowCountAndUnindexedData) { + const auto bytes = make_blob_file({"a", std::nullopt, ""}); + for (const int64_t rows : {0, 2, 4}) { + BlobFileReader reader; + EXPECT_TRUE(reader.open(std::make_shared(bytes), rows, nullptr) + .is()); + } + BlobFileReader reader; + EXPECT_TRUE(reader.open(std::make_shared(finish_blob_file( + std::string(17, '\0'), encode_record_lengths({16}))), + 1, nullptr) + .is()); +} + +TEST(BlobFileReaderTest, ValidatesMagicStoredLengthPayloadAndChecksum) { + // Corrupt one field at a time. Opening and descriptor access still succeed: records + // are deliberately read and validated only when inline output is requested. + for (const size_t corrupt_offset : {0, 9, 4, 20}) { + SCOPED_TRACE(corrupt_offset); + auto bytes = make_blob_file({"hello"}); + bytes[corrupt_offset] ^= 1; + BlobFileReader reader; + ASSERT_TRUE(reader.open(std::make_shared(bytes), 1, nullptr).ok()); + std::optional range; + ASSERT_TRUE(reader.get_range(0, &range).ok()); + ASSERT_TRUE(range.has_value()); + PODArray payload; + const auto status = reader.read_payload(*range, &payload, nullptr); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find(corrupt_offset == 0 || corrupt_offset == 9 ? "framing" + : "checksum"), + std::string::npos); + } +} + +TEST(BlobFileReaderTest, DetectsShortFooterIndexAndRecordReads) { + const auto bytes = make_blob_file({"hello"}); + for (const size_t offset : {bytes.size() - 5, bytes.size() - 6, size_t {0}}) { + SCOPED_TRACE(offset); + auto file = std::make_shared(bytes); + file->short_read_offset = offset; + BlobFileReader reader; + auto status = reader.open(file, 1, nullptr); + if (offset == 0) { + ASSERT_TRUE(status.ok()); + std::optional range; + ASSERT_TRUE(reader.get_range(0, &range).ok()); + ASSERT_TRUE(range.has_value()); + PODArray payload; + status = reader.read_payload(*range, &payload, nullptr); + } + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Truncated BLOB file"), std::string::npos); + } +} + +TEST(BlobFileReaderTest, PropagatesReadErrorsAndCancellation) { + const auto bytes = make_blob_file({"hello"}); + auto file = std::make_shared(bytes); + file->error_offset = bytes.size() - 5; + BlobFileReader reader; + EXPECT_TRUE(reader.open(file, 1, nullptr).is()); + io::IOContext context; + context.should_stop = true; + EXPECT_TRUE(reader.open(std::make_shared(bytes), 1, &context) + .is()); +} + +TEST(BlobFileReaderTest, DescribesOversizedPayloadWithoutAllocatingIt) { + const int64_t payload_size = int64_t {std::numeric_limits::max()} + 1; + const auto record_size = static_cast(payload_size + 16); + // Only the tiny tail exists in memory. Any attempted payload read fails the fake reader. + auto file = std::make_shared( + finish_blob_file("", encode_record_lengths({payload_size + 16})), record_size); + BlobFileReader reader; + ASSERT_TRUE(reader.open(file, 1, nullptr).ok()); + std::optional range; + ASSERT_TRUE(reader.get_range(0, &range).ok()); + ASSERT_TRUE(range.has_value()); + EXPECT_EQ(range->offset, 4); + EXPECT_EQ(range->size, payload_size); + PODArray payload; + EXPECT_TRUE( + reader.read_payload(*range, &payload, nullptr).is()); + EXPECT_TRUE(payload.empty()); + EXPECT_EQ(file->read_requests.size(), 2); +} + +} // namespace +} // namespace doris::format diff --git a/be/test/format_v2/blob/blob_test_util.h b/be/test/format_v2/blob/blob_test_util.h new file mode 100644 index 00000000000000..4d6eb31b49b275 --- /dev/null +++ b/be/test/format_v2/blob/blob_test_util.h @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace doris::format::blob_test { + +// Small v1 fixtures for boundary/corruption tests. The reader test also pins bytes written +// by Paimon 1.4.2, so this helper is not the only oracle for the on-disk format. +inline void append_little_endian(std::string* bytes, uint64_t value, size_t width) { + for (size_t i = 0; i < width; ++i) { + bytes->push_back(static_cast((value >> (i * 8)) & 0xff)); + } +} + +inline std::string encode_record_lengths(const std::vector& lengths) { + std::string index; + int64_t previous = 0; + for (const int64_t length : lengths) { + const int64_t delta = length - previous; + uint64_t encoded = (static_cast(delta) << 1) ^ + static_cast(-static_cast(delta < 0)); + do { + auto byte = static_cast(encoded & 0x7f); + encoded >>= 7; + index.push_back(static_cast(byte | (encoded == 0 ? 0 : 0x80))); + } while (encoded != 0); + previous = length; + } + return index; +} + +inline std::string finish_blob_file(std::string records, const std::string& index) { + records += index; + append_little_endian(&records, index.size(), 4); + records.push_back('\1'); + return records; +} + +inline std::string make_blob_file(const std::vector>& payloads) { + std::string records; + std::vector lengths; + for (const auto& payload : payloads) { + if (!payload) { + lengths.push_back(-1); + continue; + } + std::string record; + append_little_endian(&record, 1481511375, 4); + record += *payload; + const auto record_length = static_cast(payload->size()) + 16; + append_little_endian(&record, record_length, 8); + const auto checksum = crc32(0, reinterpret_cast(record.data()), + static_cast(record.size())); + append_little_endian(&record, checksum, 4); + records += record; + lengths.push_back(record_length); + } + return finish_blob_file(std::move(records), encode_record_lengths(lengths)); +} + +} // namespace doris::format::blob_test diff --git a/be/test/format_v2/blob/paimon_blob_column_reader_test.cpp b/be/test/format_v2/blob/paimon_blob_column_reader_test.cpp new file mode 100644 index 00000000000000..78ebe6d99764fa --- /dev/null +++ b/be/test/format_v2/blob/paimon_blob_column_reader_test.cpp @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/blob/paimon_blob_column_reader.h" + +#include + +#include +#include +#include + +#include "format_v2/blob/paimon_blob_test_util.h" + +namespace doris::format::paimon { +namespace { + +class PaimonBlobColumnReaderTest : public blob_test::PaimonBlobTest {}; + +TEST_F(PaimonBlobColumnReaderTest, SortsFilesAndRequiresContiguousAnchorCoverage) { + auto first = write_blob("first.blob", 7, 1000, {"a", "b", "c"}); + auto second = write_blob("second.blob", 7, 1003, {"d", "e"}); + std::vector files {&second, &first}; + ASSERT_TRUE(PaimonBlobColumnReader::validate_blob_files(&files, 1000, 5).ok()); + EXPECT_EQ(files.front(), &first); + EXPECT_EQ(files.back(), &second); + + second.__set_first_row_id(1002); + EXPECT_TRUE(PaimonBlobColumnReader::validate_blob_files(&files, 1000, 5) + .is()); + second.__set_first_row_id(1004); + second.__set_row_count(1); + EXPECT_TRUE(PaimonBlobColumnReader::validate_blob_files(&files, 1000, 5) + .is()); + second.__set_first_row_id(1003); + EXPECT_TRUE(PaimonBlobColumnReader::validate_blob_files(&files, 1000, 5) + .is()); +} + +TEST_F(PaimonBlobColumnReaderTest, RejectsMissingOrInvalidFileMetadata) { + const auto valid = write_blob("valid.blob", 7, 1000, {"a"}); + std::vector invalid(9, valid); + invalid[0].__isset.path = false; + invalid[1].__set_path(""); + invalid[2].__isset.file_size = false; + invalid[3].__set_file_size(4); + invalid[4].__isset.first_row_id = false; + invalid[5].__set_first_row_id(999); + invalid[6].__isset.row_count = false; + invalid[7].__set_row_count(0); + invalid[8].__set_row_count(2); + for (const auto& file : invalid) { + std::vector files {&file}; + EXPECT_TRUE(PaimonBlobColumnReader::validate_blob_files(&files, 1000, 1) + .is()); + } + std::vector empty; + EXPECT_TRUE(PaimonBlobColumnReader::validate_blob_files(&empty, 1000, 1) + .is()); +} + +TEST_F(PaimonBlobColumnReaderTest, ReadsSparseAndNonMonotonicPositionsAcrossFilesAndBatches) { + // Anchor starts at 1000. Its position 4 maps to logical row 1004, record 1 of second.blob. + auto first = write_blob("first.blob", 7, 1000, {"first", std::nullopt, ""}); + auto second = write_blob("second.blob", 7, 1003, {"second", std::string("\0\xff", 2)}); + std::vector files {&second, &first}; + ASSERT_TRUE(PaimonBlobColumnReader::validate_blob_files(&files, 1000, 5).ok()); + PaimonBlobColumnReader reader(blob_column("renamed_image", 7), files, false); + auto context = read_context(); + MutableColumnPtr result; + ASSERT_TRUE(reader.read_blob(*row_positions({4, 0, 2, 1, 3, 4}), 1000, context, &result).ok()); + ASSERT_EQ(result->size(), 6); + expect_inline(*result, 0, std::string("\0\xff", 2)); + expect_inline(*result, 1, "first"); + expect_inline(*result, 2, ""); + expect_inline(*result, 3, std::nullopt); + expect_inline(*result, 4, "second"); + expect_inline(*result, 5, std::string("\0\xff", 2)); + + MutableColumnPtr next_batch; + ASSERT_TRUE(reader.read_blob(*row_positions({3, 0}), 1000, context, &next_batch).ok()); + expect_inline(*next_batch, 0, "second"); + expect_inline(*next_batch, 1, "first"); + // Earlier output owns its bytes even after the next batch reuses file readers and scratch. + expect_inline(*result, 0, std::string("\0\xff", 2)); +} + +TEST_F(PaimonBlobColumnReaderTest, InlineValuesOwnLargePayloadsAndKeepMetadataNull) { + const std::string large(8193, 'x'); + auto file = write_blob("inline.blob", 7, 10, {large, "short", "", std::nullopt}); + PaimonBlobColumnReader reader(blob_column("image", 7), {&file}, false); + auto context = read_context(); + MutableColumnPtr result; + ASSERT_TRUE(reader.read_blob(*row_positions({0, 1, 2, 3}), 10, context, &result).ok()); + ASSERT_EQ(result->size(), 4); + expect_inline(*result, 0, large); + expect_inline(*result, 1, "short"); + expect_inline(*result, 2, ""); + expect_inline(*result, 3, std::nullopt); +} + +TEST_F(PaimonBlobColumnReaderTest, DescriptorValuesKeepInlineNullAndIgnoreInlineLimit) { + auto first = write_blob("first.blob", 7, 1000, {"hello", std::nullopt, ""}); + auto second = write_blob("second.blob", 7, 1003, {"world"}); + PaimonBlobColumnReader reader(blob_column("image", 7), {&first, &second}, true); + auto context = read_context(1); + MutableColumnPtr result; + ASSERT_TRUE(reader.read_blob(*row_positions({3, 2, 1, 0}), 1000, context, &result).ok()); + ASSERT_EQ(result->size(), 4); + expect_descriptor(*result, 0, second.path, 4, 5); + expect_descriptor(*result, 1, first.path, 25, 0); + expect_inline(*result, 2, std::nullopt); + expect_descriptor(*result, 3, first.path, 4, 5); +} + +TEST_F(PaimonBlobColumnReaderTest, ChecksSizeBeforeReadingPayloadAndAcceptsExactLimit) { + // The 9-byte row has an invalid CRC. A limit error proves it is rejected before payload I/O. + auto bytes = blob_test::make_blob_file({"12345678", "123456789", "", std::nullopt}); + bytes[24 + 25 - 1] ^= 1; + auto file = write_blob_bytes("limit.blob", 7, 0, 4, bytes); + PaimonBlobColumnReader reader(blob_column("image", 7), {&file}, false); + auto context = read_context(8); + MutableColumnPtr result; + ASSERT_TRUE(reader.read_blob(*row_positions({0, 2, 3}), 0, context, &result).ok()); + expect_inline(*result, 0, "12345678"); + expect_inline(*result, 1, ""); + expect_inline(*result, 2, std::nullopt); + const auto limited = reader.read_blob(*row_positions({1}), 0, context, &result); + EXPECT_TRUE(limited.is()) << limited; + EXPECT_NE(limited.to_string().find("paimon_blob_max_size=8"), std::string::npos); + context.blob_max_size = 9; + EXPECT_TRUE( + reader.read_blob(*row_positions({1}), 0, context, &result).is()); + PaimonBlobColumnReader descriptor_reader(blob_column("image", 7), {&file}, true); + ASSERT_TRUE(descriptor_reader.read_blob(*row_positions({1}), 0, context, &result).ok()); + expect_descriptor(*result, 0, file.path, 28, 9); +} + +TEST_F(PaimonBlobColumnReaderTest, MissingNullableFieldBecomesNullButRequiredFieldFails) { + auto context = read_context(); + MutableColumnPtr result; + PaimonBlobColumnReader nullable_reader(blob_column("added_image", 7), {}, false); + ASSERT_TRUE(nullable_reader.read_blob(*row_positions({2, 0}), 1000, context, &result).ok()); + ASSERT_EQ(result->size(), 2); + expect_inline(*result, 0, std::nullopt); + expect_inline(*result, 1, std::nullopt); + PaimonBlobColumnReader required_reader(blob_column("required_image", 7, false), {}, false); + EXPECT_TRUE(required_reader.read_blob(*row_positions({0}), 1000, context, &result) + .is()); +} + +TEST_F(PaimonBlobColumnReaderTest, RejectsNullInRequiredField) { + auto file = write_blob("null.blob", 7, 0, {std::nullopt}); + PaimonBlobColumnReader reader(blob_column("image", 7, false), {&file}, false); + auto context = read_context(); + MutableColumnPtr result; + EXPECT_TRUE( + reader.read_blob(*row_positions({0}), 0, context, &result).is()); +} + +TEST_F(PaimonBlobColumnReaderTest, OpensOnlySelectedFilesAndCopiesDescriptors) { + auto good = write_blob("good.blob", 7, 10, {"selected"}); + auto absent = good; + absent.__set_path((_temp_directory.path() / "does_not_exist.blob").string()); + absent.__set_first_row_id(11); + PaimonBlobColumnReader reader(blob_column("image", 7), {&good, &absent}, false); + good.__set_path("metadata_was_changed_after_construction"); + auto context = read_context(); + MutableColumnPtr result; + ASSERT_TRUE(reader.read_blob(*row_positions({}), 10, context, &result).ok()); + EXPECT_EQ(result->size(), 0); + ASSERT_TRUE(reader.read_blob(*row_positions({0}), 10, context, &result).ok()); + expect_inline(*result, 0, "selected"); + EXPECT_FALSE(reader.read_blob(*row_positions({1}), 10, context, &result).ok()); +} + +TEST_F(PaimonBlobColumnReaderTest, RejectsOverreportedFileSizeAndCancellation) { + auto file = write_blob("mismatch.blob", 7, 0, {"hello"}); + file.__set_file_size(file.file_size + 1); + PaimonBlobColumnReader reader(blob_column("image", 7), {&file}, false); + auto context = read_context(); + MutableColumnPtr result; + // Local readers use the manifest size and report EOF when it exceeds the actual file. + // Preserve that I/O error rather than requiring the BLOB parser's Corruption status. + const auto status = reader.read_blob(*row_positions({0}), 0, context, &result); + EXPECT_FALSE(status.ok()) << status; + EXPECT_NE(status.to_string().find("unexpected EOF"), std::string::npos); + _io_context.should_stop = true; + EXPECT_TRUE( + reader.read_blob(*row_positions({0}), 0, context, &result).is()); +} + +} // namespace +} // namespace doris::format::paimon diff --git a/be/test/format_v2/blob/paimon_blob_materializer_test.cpp b/be/test/format_v2/blob/paimon_blob_materializer_test.cpp new file mode 100644 index 00000000000000..6ddc72e3e8be3a --- /dev/null +++ b/be/test/format_v2/blob/paimon_blob_materializer_test.cpp @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/blob/paimon_blob_materializer.h" + +#include + +#include +#include +#include +#include +#include + +#include "core/data_type/data_type_number.h" +#include "format_v2/blob/paimon_blob_test_util.h" + +namespace doris::format::paimon { +namespace { + +class PaimonBlobMaterializerTest : public blob_test::PaimonBlobTest {}; + +TEST_F(PaimonBlobMaterializerTest, MatchesFieldIdsAfterRenameAndProjectionReordering) { + auto image_first = write_blob("image_first.blob", 7, 1000, {"image-0", std::nullopt}); + auto image_second = write_blob("image_second.blob", 7, 1002, {"", "image-3"}); + auto thumbnail = + write_blob("thumbnail.blob", 9, 1000, {"thumb-0", "thumb-1", std::nullopt, "thumb-3"}); + auto split = blob_split(1000, 4, {thumbnail, image_second, image_first}); + ColumnDefinition id_column; + id_column.name = "id"; + id_column.type = std::make_shared(); + std::vector columns {blob_column("renamed_thumbnail", 9), id_column, + blob_column("renamed_image", 7), + blob_column("added_image", 11)}; + PaimonBlobMaterializer materializer; + ASSERT_TRUE(materializer.init({}).ok()); + ASSERT_TRUE(materializer.prepare_split(split, columns).ok()); + const std::set expected_names {"renamed_thumbnail", "renamed_image", + "added_image"}; + EXPECT_EQ(materializer.blob_column_names(), expected_names); + + // Filtering/byte splitting supplies original anchor positions, not dense output ordinals. + auto positions = row_positions({3, 0, 2}); + auto block = output_block(columns, 3); + auto ids = ColumnInt32::create(); + for (const int32_t id : {103, 100, 102}) { + ids->insert_value(id); + } + block.replace_by_position(1, std::move(ids)); + auto context = read_context(); + ASSERT_TRUE(materializer.materialize(*positions, &block, context).ok()); + ASSERT_EQ(block.rows(), 3); + const auto& thumbnails = *block.get_by_position(0).column; + expect_inline(thumbnails, 0, "thumb-3"); + expect_inline(thumbnails, 1, "thumb-0"); + expect_inline(thumbnails, 2, std::nullopt); + const auto& images = *block.get_by_position(2).column; + expect_inline(images, 0, "image-3"); + expect_inline(images, 1, "image-0"); + expect_inline(images, 2, ""); + for (size_t row = 0; row < 3; ++row) { + expect_inline(*block.get_by_position(3).column, row, std::nullopt); + } + EXPECT_EQ((*block.get_by_position(1).column)[0].get(), 103); + EXPECT_EQ((*block.get_by_position(1).column)[1].get(), 100); + EXPECT_EQ((*block.get_by_position(1).column)[2].get(), 102); +} + +TEST_F(PaimonBlobMaterializerTest, SplitChangesReleaseFilesButRetainDescriptorMode) { + auto first = write_blob("first.blob", 7, 1000, {"first"}); + const std::vector columns {blob_column("image", 7)}; + PaimonBlobMaterializer materializer; + ASSERT_TRUE(materializer.init({{"blob-as-descriptor", "true"}}).ok()); + ASSERT_TRUE(materializer.prepare_split(blob_split(1000, 1, {first}), columns).ok()); + auto block = output_block(columns, 1); + auto context = read_context(1); + ASSERT_TRUE(materializer.materialize(*row_positions({0}), &block, context).ok()); + expect_descriptor(*block.get_by_position(0).column, 0, first.path, 4, 5); + + auto second = write_blob("second.blob", 7, 2000, {"second"}); + ASSERT_TRUE(materializer.prepare_split(blob_split(2000, 1, {second}), columns).ok()); + block = output_block(columns, 1); + ASSERT_TRUE(materializer.materialize(*row_positions({0}), &block, context).ok()); + expect_descriptor(*block.get_by_position(0).column, 0, second.path, 4, 6); + + // A present empty list means an added nullable field, not an ordinary non-BLOB split. + ASSERT_TRUE(materializer.prepare_split(blob_split(3000, 1, {}), columns).ok()); + EXPECT_TRUE(materializer.has_blob_columns()); + block = output_block(columns, 1); + ASSERT_TRUE(materializer.materialize(*row_positions({0}), &block, context).ok()); + expect_inline(*block.get_by_position(0).column, 0, std::nullopt); + + // An absent list disables BLOB materialization and must not retain the previous split. + ASSERT_TRUE(materializer.prepare_split(TPaimonFileDesc {}, columns).ok()); + EXPECT_FALSE(materializer.has_blob_columns()); + EXPECT_TRUE(materializer.blob_column_names().empty()); +} + +TEST_F(PaimonBlobMaterializerTest, ValidatesUnprojectedMetadataWithoutOpeningUnprojectedFiles) { + auto image = write_blob("image.blob", 7, 1000, {"a", "b", "c"}); + auto unprojected = image; + unprojected.__set_field_id(9); + unprojected.__set_path((_temp_directory.path() / "absent.blob").string()); + const std::vector columns {blob_column("image", 7)}; + PaimonBlobMaterializer materializer; + ASSERT_TRUE( + materializer.prepare_split(blob_split(1000, 3, {image, unprojected}), columns).ok()); + auto block = output_block(columns, 1); + auto context = read_context(1); + ASSERT_TRUE(materializer.materialize(*row_positions({2}), &block, context).ok()); + expect_inline(*block.get_by_position(0).column, 0, "c"); + + // Even an unprojected field must not hide a hole in its manifest row coverage. + unprojected.__set_row_count(2); + EXPECT_TRUE(materializer.prepare_split(blob_split(1000, 3, {image, unprojected}), columns) + .is()); +} + +TEST_F(PaimonBlobMaterializerTest, RejectsMissingNegativeAndOverflowingAnchorRanges) { + const auto valid = blob_split(1000, 1, {}); + std::vector invalid(7, valid); + invalid[0].__isset.anchor_first_row_id = false; + invalid[1].__isset.anchor_row_count = false; + invalid[2].__set_anchor_first_row_id(-1); + invalid[3].__set_anchor_row_count(0); + invalid[4].__set_anchor_row_count(-1); + invalid[5].__set_anchor_first_row_id(std::numeric_limits::max()); + invalid[6].__set_anchor_row_count(std::numeric_limits::max()); + PaimonBlobMaterializer materializer; + const std::vector columns {blob_column("image", 7)}; + for (const auto& split : invalid) { + EXPECT_TRUE(materializer.prepare_split(split, columns).is()); + } + EXPECT_TRUE(materializer + .prepare_split(blob_split(std::numeric_limits::max() - 1, 1, {}), + columns) + .ok()); +} + +TEST_F(PaimonBlobMaterializerTest, RejectsMissingOrInvalidFieldIds) { + auto file = write_blob("image.blob", 7, 0, {"a"}); + PaimonBlobMaterializer materializer; + const std::vector columns {blob_column("image", 7)}; + file.__isset.field_id = false; + EXPECT_TRUE(materializer.prepare_split(blob_split(0, 1, {file}), columns) + .is()); + file.__set_field_id(-1); + EXPECT_TRUE(materializer.prepare_split(blob_split(0, 1, {file}), columns) + .is()); + auto no_field_id = blob_column("image", 7); + no_field_id.identifier = Field(); + EXPECT_TRUE(materializer.prepare_split(blob_split(0, 1, {}), {no_field_id}) + .is()); +} + +TEST_F(PaimonBlobMaterializerTest, RejectsMisalignedAndOutOfRangePositionsBeforeReadingFiles) { + const std::vector columns {blob_column("image", 7)}; + auto absent = write_blob("image.blob", 7, 1000, {"a", "b"}); + absent.__set_path((_temp_directory.path() / "absent.blob").string()); + PaimonBlobMaterializer materializer; + ASSERT_TRUE(materializer.prepare_split(blob_split(1000, 2, {absent}), columns).ok()); + auto context = read_context(); + auto block = output_block(columns, 2); + EXPECT_TRUE(materializer.materialize(*row_positions({0}), &block, context) + .is()); + for (const int64_t invalid_position : {-1, 2}) { + block = output_block(columns, 1); + EXPECT_TRUE(materializer.materialize(*row_positions({invalid_position}), &block, context) + .is()); + } + block = output_block(columns, 0); + EXPECT_TRUE(materializer.materialize(*row_positions({}), &block, context).ok()); +} + +TEST_F(PaimonBlobMaterializerTest, NoProjectedBlobColumnsLeaveOrdinaryOutputUnchanged) { + ColumnDefinition id; + id.name = "id"; + id.type = std::make_shared(); + auto unprojected = write_blob("image.blob", 7, 0, {"hello"}); + unprojected.__set_path((_temp_directory.path() / "absent.blob").string()); + PaimonBlobMaterializer materializer; + ASSERT_TRUE(materializer.prepare_split(blob_split(0, 1, {unprojected}), {id}).ok()); + EXPECT_FALSE(materializer.has_blob_columns()); + auto block = output_block({id}, 1); + auto context = read_context(1); + EXPECT_TRUE(materializer.materialize(*row_positions({0}), &block, context).ok()); + ASSERT_EQ(block.columns(), 1); + EXPECT_EQ((*block.get_by_position(0).column)[0].get(), 0); +} + +TEST_F(PaimonBlobMaterializerTest, ValidatesDescriptorOptionAndRestoresInlineDefaultOnInit) { + PaimonBlobMaterializer materializer; + for (const auto* value : {"", "TRUE", "1", "yes"}) { + EXPECT_TRUE(materializer.init({{"blob-as-descriptor", value}}) + .is()); + } + ASSERT_TRUE(materializer.init({{"blob-as-descriptor", "true"}}).ok()); + ASSERT_TRUE(materializer.init({}).ok()); + auto file = write_blob("inline.blob", 7, 0, {"hello"}); + const std::vector columns {blob_column("image", 7)}; + ASSERT_TRUE(materializer.prepare_split(blob_split(0, 1, {file}), columns).ok()); + auto block = output_block(columns, 1); + auto context = read_context(); + ASSERT_TRUE(materializer.materialize(*row_positions({0}), &block, context).ok()); + expect_inline(*block.get_by_position(0).column, 0, "hello"); +} + +} // namespace +} // namespace doris::format::paimon diff --git a/be/test/format_v2/blob/paimon_blob_test_util.h b/be/test/format_v2/blob/paimon_blob_test_util.h new file mode 100644 index 00000000000000..18ae9dbf3b22a1 --- /dev/null +++ b/be/test/format_v2/blob/paimon_blob_test_util.h @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "core/block/block.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_nullable.h" +#include "format_v2/blob/blob_test_util.h" +#include "format_v2/blob/paimon_blob_column_reader.h" +#include "io/file_factory.h" +#include "io/io_common.h" +#include "testutil/scoped_temp_dir.h" + +namespace doris::format::blob_test { + +// Use real local FileFactory reads without a JVM, object store, or a Parquet scan. +class PaimonBlobTest : public ::testing::Test { +protected: + PaimonBlobTest() { _system_properties.system_type = TFileType::FILE_LOCAL; } + + TPaimonBlobFileDesc write_blob(const std::string& name, int32_t field_id, int64_t first_row_id, + const std::vector>& payloads) { + return write_blob_bytes(name, field_id, first_row_id, static_cast(payloads.size()), + make_blob_file(payloads)); + } + + TPaimonBlobFileDesc write_blob_bytes(const std::string& name, int32_t field_id, + int64_t first_row_id, int64_t row_count, + const std::string& bytes) { + const auto path = (_temp_directory.path() / name).string(); + std::ofstream out(path, std::ios::binary); + out.write(bytes.data(), static_cast(bytes.size())); + out.close(); + EXPECT_TRUE(out.good()) << path; + TPaimonBlobFileDesc file; + file.__set_field_id(field_id); + file.__set_path(path); + file.__set_file_size(static_cast(bytes.size())); + file.__set_first_row_id(first_row_id); + file.__set_row_count(row_count); + return file; + } + + static ColumnDefinition blob_column(std::string name, int32_t field_id, bool nullable = true) { + ColumnDefinition column; + column.name = std::move(name); + column.identifier = Field::create_field(field_id); + column.type = std::make_shared(); + if (nullable) { + column.type = make_nullable(column.type); + } + return column; + } + + static auto row_positions(std::initializer_list positions) { + auto column = ColumnInt64::create(); + for (const auto position : positions) { + column->insert_value(position); + } + return column; + } + + paimon::PaimonBlobColumnReader::ReadContext read_context(int64_t max_size = 10485760) { + return {.system_properties = _system_properties, + .runtime_state = nullptr, + .io_ctx = &_io_context, + .fs_name = _fs_name, + .file_cache_admission = false, + .blob_max_size = max_size}; + } + + static TPaimonFileDesc blob_split(int64_t anchor_first_row_id, int64_t anchor_row_count, + std::vector files) { + TPaimonFileDesc split; + split.__set_anchor_first_row_id(anchor_first_row_id); + split.__set_anchor_row_count(anchor_row_count); + split.__set_blob_files(std::move(files)); + return split; + } + + static Block output_block(const std::vector& columns, size_t rows) { + Block block; + for (const auto& column : columns) { + auto data = column.type->create_column(); + data->insert_many_defaults(rows); + block.insert({std::move(data), column.type, column.name}); + } + return block; + } + + static void expect_inline(const IColumn& column, size_t row, + const std::optional& expected) { + const auto value = column[row]; + if (!expected) { + EXPECT_TRUE(value.is_null()) << "row " << row; + return; + } + ASSERT_FALSE(value.is_null()) << "row " << row; + const auto& fields = value.get(); + ASSERT_EQ(fields.size(), DataTypeFile::FIELD_COUNT); + for (size_t child = 0; child < DataTypeFile::INLINE_FIELD_INDEX; ++child) { + EXPECT_TRUE(fields[child].is_null()) << "row " << row << " child " << child; + } + ASSERT_FALSE(fields[DataTypeFile::INLINE_FIELD_INDEX].is_null()); + const auto& bytes = fields[DataTypeFile::INLINE_FIELD_INDEX].get(); + EXPECT_EQ(std::string(bytes.data(), bytes.size()), *expected) << "row " << row; + } + + static void expect_descriptor(const IColumn& column, size_t row, const std::string& path, + int64_t offset, int64_t size) { + const auto value = column[row]; + ASSERT_FALSE(value.is_null()) << "row " << row; + const auto& fields = value.get(); + ASSERT_EQ(fields.size(), DataTypeFile::FIELD_COUNT); + ASSERT_FALSE(fields[DataTypeFile::URI_FIELD_INDEX].is_null()); + ASSERT_FALSE(fields[DataTypeFile::OFFSET_FIELD_INDEX].is_null()); + ASSERT_FALSE(fields[DataTypeFile::SIZE_FIELD_INDEX].is_null()); + EXPECT_EQ(fields[DataTypeFile::URI_FIELD_INDEX].get(), path); + EXPECT_EQ(fields[DataTypeFile::OFFSET_FIELD_INDEX].get(), offset); + EXPECT_EQ(fields[DataTypeFile::SIZE_FIELD_INDEX].get(), size); + EXPECT_TRUE(fields[DataTypeFile::INLINE_FIELD_INDEX].is_null()); + EXPECT_TRUE(fields[DataTypeFile::CONTENT_TYPE_FIELD_INDEX].is_null()); + EXPECT_TRUE(fields[DataTypeFile::CHECKSUM_FIELD_INDEX].is_null()); + } + + doris::test::ScopedTempDirectory _temp_directory {"paimon_blob"}; + io::FileSystemProperties _system_properties; + io::IOContext _io_context; + std::string _fs_name; +}; + +} // namespace doris::format::blob_test diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql new file mode 100644 index 00000000000000..2437a454024b01 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql @@ -0,0 +1,262 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +use paimon; +create database if not exists test_paimon_spark_blob; +use test_paimon_spark_blob; + +-- Default blob storage fixtures for Doris FileScannerV2 native/JNI reads. +-- Spark exposes BLOB as BINARY; blob-field converts the named columns to Paimon BLOB. +-- Do not use blob-descriptor-field or blob-external-storage-field here. +-- See blob_fixtures.md for the expected results and suggested Groovy assertions. +-- Binary values use hex/UTF-8 encoding explicitly; no CHAR/VARCHAR casts can truncate them. +create or replace temporary view paimon_blob_seed as +select id, case_name, image, length(image) as expected_size, hex(image) as expected_hex +from values + (1, 'null_first', cast(null as binary)), + (2, 'empty', unhex('')), + (3, 'one_zero', unhex('00')), + (4, 'one_ascii_max', unhex('7F')), + (5, 'one_high_bit', unhex('80')), + (6, 'one_ff', unhex('FF')), + (7, 'embedded_zero', unhex('004100FF')), + (8, 'all_byte_values', unhex(concat( + '000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F', + '202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F', + '404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F', + '606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F', + '808182838485868788898A8B8C8D8E8F909192939495969798999A9B9C9D9E9F', + 'A0A1A2A3A4A5A6A7A8A9AAABACADAEAFB0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF', + 'C0C1C2C3C4C5C6C7C8C9CACBCCCDCECFD0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF', + 'E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEFF0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF'))), + (9, 'utf8', encode('你好Doris🌍', 'UTF-8')), + (10, 'hello_world', unhex('68656C6C6F20776F726C64')), + (11, 'limit_minus_one', encode('1234567', 'UTF-8')), + (12, 'limit_exact', encode('12345678', 'UTF-8')), + (13, 'limit_plus_one', encode('123456789', 'UTF-8')), + (14, 'json', encode('{"id":1,"name":"blob","active":true}', 'UTF-8')), + (15, 'uri_is_payload', encode('s3://not-a-reference/image.bin', 'UTF-8')), + (16, 'control_bytes', unhex('00090A0D1B7FFF')), + (17, 'repeated_payload', unhex('68656C6C6F20776F726C64')), + (18, 'null_middle', cast(null as binary)), + (19, 'empty_after_null', unhex('')), + (20, 'length_1', encode('a', 'UTF-8')), + (21, 'length_47', encode(repeat('a', 47), 'UTF-8')), + (22, 'length_48', encode(repeat('b', 48), 'UTF-8')), + (23, 'length_49', encode(repeat('c', 49), 'UTF-8')), + (24, 'length_111', encode(repeat('a', 111), 'UTF-8')), + (25, 'length_112', encode(repeat('b', 112), 'UTF-8')), + (26, 'length_113', encode(repeat('c', 113), 'UTF-8')), + (27, 'length_127', encode(repeat('a', 127), 'UTF-8')), + (28, 'length_128', encode(repeat('b', 128), 'UTF-8')), + (29, 'length_129', encode(repeat('c', 129), 'UTF-8')), + (30, 'length_255', encode(repeat('a', 255), 'UTF-8')), + (31, 'length_256', encode(repeat('b', 256), 'UTF-8')), + (32, 'length_257', encode(repeat('c', 257), 'UTF-8')), + (33, 'length_8175', encode(repeat('a', 8175), 'UTF-8')), + (34, 'empty_after_large', unhex('')), + (35, 'length_8176', encode(repeat('b', 8176), 'UTF-8')), + (36, 'null_after_large', cast(null as binary)), + (37, 'length_8177', encode(repeat('c', 8177), 'UTF-8')), + (38, 'small_after_large', unhex('78')), + (39, 'null_last', cast(null as binary)) +as seed(id, case_name, image); + +-- One ordered writer keeps NULL/empty/large/small transitions in the same .blob file. +-- The 8176-byte transitions also exercise three-byte zigzag varints without large objects. +drop table if exists blob_basic; +create table blob_basic ( + id int, case_name string, image binary, expected_size int, expected_hex string +) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); +insert into blob_basic +select /*+ COALESCE(1) */ * from paimon_blob_seed order by id; + +drop table if exists blob_empty; +create table blob_empty (id int, image binary) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); + +-- Nonempty table, but the BLOB has no non-NULL records: index/footer-only .blob files. +drop table if exists blob_all_null; +create table blob_all_null (id int, image binary) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); +insert into blob_all_null +select /*+ COALESCE(1) */ cast(id as int), cast(null as binary) from range(1, 5, 1, 1); + +-- A small, isolated fixture for paimon_blob_max_size=8: NULL, 0, 7, 8 and 9 bytes. +-- Test inline failures separately from descriptor output and from projections of id alone. +drop table if exists blob_size_limit; +create table blob_size_limit ( + id int, case_name string, image binary, expected_size int, expected_hex string +) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); +insert into blob_size_limit +select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (1, 2, 11, 12, 13) order by id; + +-- Two BLOB fields separated by an ordinary column. NULL maps and sizes are independent. +create or replace temporary view paimon_blob_pair_seed as +select id, image, note, thumbnail, + length(image) as expected_size, hex(image) as expected_hex, + length(thumbnail) as expected_thumbnail_size, hex(thumbnail) as expected_thumbnail_hex +from values + (1, cast(null as binary), 'both_null', cast(null as binary)), + (2, unhex(''), 'both_empty', unhex('')), + (3, unhex('68656C6C6F'), 'right_null', cast(null as binary)), + (4, cast(null as binary), 'left_null', unhex('776F726C64')), + (5, unhex('00FF'), 'different', unhex('68656C6C6F20776F726C64')), + (6, unhex(''), 'left_empty', unhex('FF0080')), + (7, unhex('FF0080'), 'right_empty', unhex('')), + (8, encode(repeat('x', 257), 'UTF-8'), 'reuse_buffer', unhex('01')) +as seed(id, image, note, thumbnail); + +drop table if exists blob_multi_columns; +create table blob_multi_columns ( + id int, image binary, note string, thumbnail binary, + expected_size int, expected_hex string, + expected_thumbnail_size int, expected_thumbnail_hex string +) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image,thumbnail', 'write-only' = 'true' +); +insert into blob_multi_columns +select /*+ COALESCE(1) */ * from paimon_blob_pair_seed order by id; + +-- Partition predicates must retain the right original row positions, including NULL partitions. +drop table if exists blob_partitioned; +create table blob_partitioned ( + id int, case_name string, image binary, expected_size int, expected_hex string, region string +) using paimon partitioned by (region) tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); +insert into blob_partitioned +select /*+ COALESCE(1) */ *, + case when id <= 6 then 'a' when id <= 12 then 'b' else cast(null as string) end +from paimon_blob_seed where id <= 13 order by id; + +-- Three commits produce nonzero anchor origins. Duplicate SQL ids remain separate append rows. +drop table if exists blob_multi_snapshot; +create table blob_multi_snapshot ( + id int, case_name string, image binary, expected_size int, expected_hex string +) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); +insert into blob_multi_snapshot +select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (1, 2, 10) order by id; +insert into blob_multi_snapshot +select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (7, 12) order by id; +insert into blob_multi_snapshot values + (10, 'duplicate_id_new_payload', unhex('6E6577'), 3, '6E6577'); + +-- More than a default scanner batch, with independently rolling companion files for two fields. +-- Each payload is at most 106 bytes. A single Parquet anchor has a much larger target than BLOBs. +-- Do not assert a fixed file count: verify rollover through $files and descriptor URI changes. +create or replace temporary view paimon_blob_rollover_seed as +select cast(id as int) as id, + case when id % 17 = 0 then cast(null as binary) + when id % 17 = 1 then unhex('') + else encode(concat('row-', lpad(cast(id as string), 5, '0'), ':', + repeat('a', cast(id % 97 as int))), 'UTF-8') end as image, + case when id % 13 = 0 then cast(null as binary) + else encode(concat('thumb-', cast(id as string)), 'UTF-8') end as thumbnail +from range(0, 4097, 1, 1); + +drop table if exists blob_rollover; +create table blob_rollover ( + id int, image binary, thumbnail binary, + expected_size int, expected_hex string, + expected_thumbnail_size int, expected_thumbnail_hex string +) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image,thumbnail', + 'target-file-size' = '64 mb', 'blob.target-file-size' = '16 kb', 'write-only' = 'true' +); +insert into blob_rollover +select /*+ COALESCE(1) */ id, image, thumbnail, + length(image), hex(image), length(thumbnail), hex(thumbnail) +from paimon_blob_rollover_seed order by id; + +-- Old/new files use different ordinary-column names and BLOB ordinals; field IDs remain stable. +-- Adding an ordinary column also covers NULL defaults in historical Parquet anchors. +drop table if exists blob_schema_evolution; +create table blob_schema_evolution ( + id int, case_name string, image binary, expected_size int, expected_hex string +) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); +insert into blob_schema_evolution +select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (1, 10) order by id; +-- Paimon 1.4.2 rejects renaming a BLOB column, so rename an ordinary column instead. +alter table blob_schema_evolution rename column case_name to label; +alter table blob_schema_evolution add columns (extra string); +alter table blob_schema_evolution alter column image first; +insert into blob_schema_evolution (image, id, label, expected_size, expected_hex, extra) +select /*+ COALESCE(1) */ image, id, case_name, expected_size, expected_hex, 'after-schema-change' +from paimon_blob_seed where id in (2, 12) order by id; + +-- Compaction rewrites ordinary anchors and can leave older BLOB companions in the row-range group. +-- Latest field-bunch selection must not duplicate or lose rows. File counts are not an oracle. +drop table if exists blob_compacted; +create table blob_compacted ( + id int, case_name string, image binary, expected_size int, expected_hex string +) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image' +); +insert into blob_compacted +select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (1, 2, 10) order by id; +insert into blob_compacted +select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (7, 12, 13) order by id; +call sys.compact(table => 'blob_compacted'); + +-- Managed BLOB with an ORC ordinary anchor: native BLOB planning must fall back to JNI. +drop table if exists blob_orc_fallback; +create table blob_orc_fallback ( + id int, case_name string, image binary, expected_size int, expected_hex string +) using paimon tblproperties ( + 'file.format' = 'orc', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); +insert into blob_orc_fallback +select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (1, 2, 7, 10, 12, 13) order by id; + +-- Change the READ default only after writing ordinary payload bytes. A relation-level +-- blob-as-descriptor=false override must still restore inline output and its size limit. +drop table if exists blob_descriptor_default; +create table blob_descriptor_default ( + id int, case_name string, image binary, expected_size int, expected_hex string +) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); +insert into blob_descriptor_default +select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (1, 2, 10) order by id; +alter table blob_descriptor_default set tblproperties ('blob-as-descriptor' = 'true'); + +drop view paimon_blob_seed; +drop view paimon_blob_pair_seed; +drop view paimon_blob_rollover_seed; diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index a5264642a7ddf6..3c678e7f7df6d0 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Blob; import org.apache.paimon.data.BlobDescriptor; +import org.apache.paimon.data.BlobRef; import org.apache.paimon.data.DataGetters; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalArray; @@ -53,6 +54,7 @@ public class PaimonColumnValue implements ColumnValue { private static final Logger LOG = LoggerFactory.getLogger(PaimonColumnValue.class); + static final long DEFAULT_BLOB_MAX_SIZE = 10L * 1024 * 1024; private static final Map DORIS_TIME_ZONE_ALIASES; // JNI transports Doris FILE as a six-child STRUCT. This is only the adapter's // child schema; the Paimon reader must continue to read the logical BLOB type. @@ -85,6 +87,7 @@ public class PaimonColumnValue implements ColumnValue { // rebuild subpath variant to doris private Variant materializedVariant; private boolean blobAsDescriptor; + private long blobMaxSize = DEFAULT_BLOB_MAX_SIZE; private GenericRow materializedBlob; // Keep these caches lazy so scalar columns do not pay for complex-type reuse bookkeeping. private List arrayValues; @@ -139,6 +142,10 @@ public void setBlobAsDescriptor(boolean blobAsDescriptor) { this.blobAsDescriptor = blobAsDescriptor; } + public void setBlobMaxSize(long blobMaxSize) { + this.blobMaxSize = blobMaxSize; + } + @Override public boolean canGetStringAsBytes() { return true; @@ -334,11 +341,33 @@ private InternalRow getBlobFileRow() { materializedBlob.setField(1, descriptor.offset()); materializedBlob.setField(2, descriptor.length() < 0 ? null : descriptor.length()); } else { - materializedBlob.setField(5, blob.toData()); + materializedBlob.setField(5, readBlobPayload(blob)); } return materializedBlob; } + private byte[] readBlobPayload(Blob blob) { + if (blob instanceof BlobRef) { + // Managed .blob references carry their length, so reject before opening the payload. + // External references can have length -1 and are checked after reading below. + checkBlobSize(blob.toDescriptor().length()); + } + // TODO: Use bounded streaming reads for unknown-length BLOBs so the size limit can be + // enforced before toData() allocates and reads the entire payload. + byte[] payload = blob.toData(); + checkBlobSize(payload.length); + return payload; + } + + private void checkBlobSize(long size) { + if (size > blobMaxSize) { + throw new IllegalArgumentException("Paimon BLOB column '" + dorisType.getName() + + "' inline size " + size + " bytes exceeds paimon_blob_max_size=" + blobMaxSize + + " bytes. Increase paimon_blob_max_size " + + "or use blob-as-descriptor=true"); + } + } + private void clearMaterializedBlob() { if (materializedBlob != null) { // VectorColumn consumes children synchronously. Drop the prior payload as soon as @@ -365,6 +394,7 @@ private PaimonColumnValue reuseColumnValue( value.reset(childRecord, childIndex, childDorisType, childPaimonType, timeZone); } value.blobAsDescriptor = blobAsDescriptor; + value.blobMaxSize = blobMaxSize; return value; } diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 8e41475525922b..b5b174847c90f1 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -40,6 +40,7 @@ import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.TableRead; import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.types.BlobType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; @@ -155,6 +156,8 @@ public PaimonJniScanner(int batchSize, Map params) { "Missing required Paimon scanner parameter: serialized_table_cache_key"); timeZone = params.getOrDefault("time_zone", TimeZone.getDefault().getID()); columnValue.setTimeZone(timeZone); + columnValue.setBlobMaxSize(Long.parseLong(params.getOrDefault("paimon_blob_max_size", + Long.toString(PaimonColumnValue.DEFAULT_BLOB_MAX_SIZE)))); this.variantAccessPathsByColumn = variantAccessPathsByColumn(params, requiredFields.length); initTableInfo(columnTypes, requiredFields, batchSize); hadoopOptionParams = params.entrySet().stream() @@ -195,10 +198,8 @@ public void open() throws IOException { } private void initReader() throws IOException { - // Read the same effective option that the SDK uses, including table-cache hits and - // relation-level overrides carried in serialized_table. + // Preserve the requested output mode, including table-cache hits and relation overrides. columnValue.setBlobAsDescriptor(new Options(table.options()).get(CoreOptions.BLOB_AS_DESCRIPTOR)); - ReadBuilder readBuilder = table.newReadBuilder(); if (this.fields.length > this.paimonAllFieldNames.size()) { throw new IOException( String.format( @@ -230,6 +231,18 @@ private void initReader() throws IOException { projection.readType().copy(tableField.type().isNullable()))); } } + Table readTable = table; + if (readFields.stream().anyMatch(field -> field.type() instanceof BlobType)) { + // The SDK otherwise allocates each managed BLOB before returning its row. Request + // references so PaimonColumnValue can enforce the size limit before reading inline. + // Keep this copy scanner-local: cached table options still describe the output mode. + Map blobReadOptions = Collections.singletonMap( + CoreOptions.BLOB_AS_DESCRIPTOR.key(), "true"); + readTable = table instanceof FileStoreTable + ? ((FileStoreTable) table).copyWithoutTimeTravel(blobReadOptions) + : table.copy(blobReadOptions); + } + ReadBuilder readBuilder = readTable.newReadBuilder(); if (hasReadTypeProjection) { // For static complex types, the recursively pruned read type tells Paimon which nested // ROW/ARRAY/MAP fields to read. For Variant, Paimon recognizes a metadata-marked RowType diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 423976177d56de..92f6629e601578 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -57,6 +57,7 @@ import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TPaimonBlobFileDesc; import org.apache.doris.thrift.TPaimonDeletionFileDesc; import org.apache.doris.thrift.TPaimonFileDesc; import org.apache.doris.thrift.TPaimonReaderType; @@ -71,7 +72,9 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.format.OrcOptions; +import org.apache.paimon.format.blob.BlobFileFormat; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.operation.DataEvolutionSplitRead; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.rest.RESTTokenFileIO; @@ -90,6 +93,7 @@ import org.apache.paimon.table.source.TableScan; import org.apache.paimon.table.source.snapshot.SnapshotReader; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.MapType; @@ -256,6 +260,10 @@ public PaimonScanNode(PlanNodeId id, @Override protected void doInitialize() throws UserException { + if (!sessionVariable.enableFileScannerV2 && hasProjectedFileColumn()) { + throw new UserException("Paimon BLOB columns require enable_file_scanner_v2=true" + + " for both native and JNI readers"); + } checkVariantV2Enabled(desc); Optional relationSnapshot = getRelationSnapshot(); if (desc.getTable() instanceof PaimonExternalTable @@ -284,6 +292,10 @@ protected void doInitialize() throws UserException { ); backendStorageProperties = CredentialUtils.getBackendPropertiesFromStorageMap(storagePropertiesMap); backendPaimonOptions = new HashMap<>(getBackendPaimonOptions()); + // Use the same effective table as serialized_table, including relation/catalog overrides + // and Paimon's default, so native readers and the JNI scanner receive one read mode. + backendPaimonOptions.put(CoreOptions.BLOB_AS_DESCRIPTOR.key(), + String.valueOf(new Options(processedTable.options()).get(CoreOptions.BLOB_AS_DESCRIPTOR))); OptionalInt manifestCap; if (source.getExternalTable() instanceof PaimonSysExternalTable) { manifestCap = source.runtimeSafeManifestParallelism(getScanParams()); @@ -1107,6 +1119,13 @@ && splitHasOrcFile((DataSplit) split) putHistorySchemaInfo(paimonSplit.getSchemaId()); fileDesc.setSchemaId(paimonSplit.getSchemaId()); } + if (paimonSplit.getBlobFiles() != null) { + // The reader remains PAIMON_NATIVE/PARQUET. These are companion column files, + // materialized by PaimonReader using the anchor's original file row positions. + fileDesc.setBlobFiles(paimonSplit.getBlobFiles()); + fileDesc.setAnchorFirstRowId(paimonSplit.getAnchorFirstRowId()); + fileDesc.setAnchorRowCount(paimonSplit.getAnchorRowCount()); + } fileDesc.setFileFormat(fileFormat); // Hadoop conf is set at ScanNode level via params.properties in createScanRangeLocations(), // no need to set it for each split to avoid redundant configuration @@ -1157,15 +1176,21 @@ protected List getDeleteFiles(TFileRangeDesc rangeDesc) { @Override public List getSplits(int numBackends) throws UserException { - // BLOB columns are materialized as FILE by the JNI adapter. Keep their logical - // Paimon splits intact instead of converting them to independent raw-file reads. - boolean forceJniScanner = sessionVariable.isForceJniScanner() || hasProjectedFileColumn(); + // BLOB columns need either the companion-file plan below or a logical JNI split. + // Ordinary raw-file conversion cannot reconstruct their FILE values. + boolean forceJniScanner = sessionVariable.isForceJniScanner(); + boolean hasProjectedFile = hasProjectedFileColumn(); // Paimon system tables need Paimon-side semantics: // - binlog: pack/merge + array materialization // - audit_log: rowkind / sequence-number projection // TODO: Allow native reader after Doris native parquet/orc reader can materialize // these system-table rows consistently with Paimon system-table semantics. boolean forceJniForSystemTable = shouldForceJniForSystemTable(); + boolean materializeBlobColumns = !forceJniScanner && !forceJniForSystemTable + && needsNativeBlobMaterialization(hasProjectedFile); + Set projectedNames = materializeBlobColumns ? desc.getSlots().stream() + .map(slot -> slot.getColumn().getName().toLowerCase(Locale.ROOT)).collect(Collectors.toSet()) + : Collections.emptySet(); SessionVariable.IgnoreSplitType ignoreSplitType = SessionVariable.IgnoreSplitType .valueOf(sessionVariable.getIgnoreSplitType()); List splits = new ArrayList<>(); @@ -1228,6 +1253,16 @@ public List getSplits(int numBackends) throws UserException { // must not pay the planning cost of merging Paimon manifest statistics. OptionalLong mergedRowCount = applyCountPushdown ? dataSplit.mergedRowCount() : OptionalLong.empty(); + Optional> blobSplits = Optional.empty(); + if (!mergedRowCount.isPresent() && materializeBlobColumns + && canUseNativeBlobSplit(dataSplit, projectedNames)) { + if (!hasDeterminedTargetFileSplitSize) { + targetFileSplitSize = determineTargetFileSplitSize(dataSplits, isBatchMode()); + hasDeterminedTargetFileSplitSize = true; + } + blobSplits = createNativeBlobSplits( + dataSplit, partitionInfoMap, targetFileSplitSize, projectedNames); + } if (applyCountPushdown && mergedRowCount.isPresent()) { long count = mergedRowCount.getAsLong(); splitStat.setMergedRowCount(count); @@ -1238,7 +1273,15 @@ public List getSplits(int numBackends) throws UserException { } pushDownCountSplits.add(split); pushDownCountSum += count; - } else if (!forceJniScanner && !forceJniForSystemTable && supportNativeReader(optRawFiles)) { + } else if (blobSplits.isPresent()) { + if (ignoreSplitType == SessionVariable.IgnoreSplitType.IGNORE_NATIVE) { + continue; + } + splits.addAll(blobSplits.get()); + rawFileSplitNum += blobSplits.get().size(); + splitStat.setType(SplitReadType.NATIVE); + } else if (!forceJniScanner && !forceJniForSystemTable && !hasProjectedFile + && supportNativeReader(optRawFiles)) { // Keep raw-file reads ahead of logical JNI/Rust dispatch: only the native file // reader can use footer aggregates for COUNT(col) and MIN/MAX. Logical DataSplits // may require merge/delete semantics and cannot reuse independent file statistics. @@ -1321,6 +1364,174 @@ public List getSplits(int numBackends) throws UserException { return splits; } + private boolean needsNativeBlobMaterialization(boolean hasProjectedFile) { + if (!hasProjectedFile || !sessionVariable.enableFileScannerV2 + || !(processedTable instanceof FileStoreTable) + || processedTable instanceof FallbackReadFileStoreTable) { + return false; + } + FileStoreTable table = (FileStoreTable) processedTable; + if (!table.coreOptions().dataEvolutionEnabled() || table.coreOptions().queryAuthEnabled()) { + return false; + } + // Nested FILE values still use JNI. Only top-level managed BLOBs can be joined to + // an anchor by its original row positions. + return desc.getSlots().stream() + .noneMatch(slot -> !slot.getType().isFileType() && slot.getType().typeContainsFile()); + } + + private boolean canUseNativeBlobSplit(DataSplit dataSplit, Set projectedNames) throws UserException { + FileStoreTable table = (FileStoreTable) processedTable; + if (dataSplit instanceof FallbackReadFileStoreTable.FallbackSplit + || !table.primaryKeys().isEmpty() || dataSplit.isStreaming()) { + return false; + } + // TODO: If Paimon enables deletion vectors for data-evolution tables, propagate each + // anchor's deletion file to its native splits and preserve original row positions + // when materializing BLOBs. Paimon 1.4.2 currently rejects this option combination. + if (dataSplit.deletionFiles().orElse(Collections.emptyList()).stream().anyMatch(Objects::nonNull)) { + return false; + } + if (table.coreOptions().blobDescriptorField().stream() + .anyMatch(name -> projectedNames.contains(name.toLowerCase(Locale.ROOT)))) { + return false; + } + for (DataFileMeta file : dataSplit.dataFiles()) { + // Companion files must use the same backend configuration as the anchor. + if (file.externalPath().isPresent() || file.externalPathDir().isPresent()) { + return false; + } + if (file.firstRowId() == null || file.firstRowId() < 0 || file.rowCount() <= 0 + || file.firstRowId() > Long.MAX_VALUE - file.rowCount()) { + throw new UserException("Missing or invalid Paimon data-evolution row range for " + file.fileName()); + } + if (BlobFileFormat.isBlobFile(file.fileName()) + && (file.writeCols() == null || file.writeCols().size() != 1)) { + throw new UserException("Managed Paimon BLOB files must contain exactly one column"); + } + } + return true; + } + + /** + * Plans native reads only for projected BLOB columns using Paimon's Default blob storage: + * payloads live in managed .blob files and are joined to ordinary columns by row ID. + * Descriptor-only and external-storage descriptor modes fall back to JNI. + * The blob-as-descriptor read option is independent of the storage mode; native reads + * support both inline payload and descriptor output for Default blob storage. + * Returns Optional.empty() when the file layout requires JNI. + */ + private Optional> createNativeBlobSplits(DataSplit dataSplit, Map partitionValues, + long targetSplitSize, Set projectedNames) throws UserException { + Set projectedBlobIds = processedTable.rowType().getFields().stream() + .filter(field -> field.type().getTypeRoot() == DataTypeRoot.BLOB + && projectedNames.contains(field.name().toLowerCase(Locale.ROOT))) + .map(DataField::id).collect(Collectors.toSet()); + List result = new ArrayList<>(); + try { + // Reuse the SDK's overlap grouping and latest-version selection. Flattening the + // DataSplit would duplicate rows, and selecting every .blob file would read obsolete + // versions after compaction. + for (List group : DataEvolutionSplitRead.mergeRangesAndSort(dataSplit.dataFiles())) { + DataFileMeta anchor = findBlobParquetAnchor(group, projectedBlobIds); + if (anchor == null) { + return Optional.empty(); + } + List bunches = DataEvolutionSplitRead.splitFieldBunches( + group, file -> getHistoryTableSchema(file.schemaId()).logicalRowType()); + List blobFiles = new ArrayList<>(); + for (DataEvolutionSplitRead.FieldBunch bunch : bunches) { + DataFileMeta first = bunch.files().get(0); + if (!BlobFileFormat.isBlobFile(first.fileName())) { + continue; + } + DataField field = getHistoryTableSchema(first.schemaId()).logicalRowType() + .getField(first.writeCols().get(0)); + if (!projectedBlobIds.contains(field.id())) { + continue; + } + if (bunch.rowCount() != anchor.rowCount() + || first.nonNullFirstRowId() != anchor.nonNullFirstRowId()) { + return Optional.empty(); + } + for (DataFileMeta blob : bunch.files()) { + blobFiles.add(createBlobFileDescriptor(dataSplit, blob)); + } + } + String path = paimonDataFilePath(dataSplit, anchor); + List physicalSplits = fileSplitter.splitFile( + LocationPath.of(path, storagePropertiesMap), + selectFeSplitSizeForRawFile(path, targetSplitSize, true), + null, anchor.fileSize(), -1, true, Collections.emptyList(), + PaimonSplit.PaimonSplitCreator.DEFAULT); + for (Split physicalSplit : physicalSplits) { + PaimonSplit split = (PaimonSplit) physicalSplit; + split.setSchemaId(anchor.schemaId()); + split.setPaimonPartitionValues(partitionValues); + // Preserve the complete anchor's row origin even when its bytes are split. + split.setBlobFiles(blobFiles, anchor.nonNullFirstRowId(), anchor.rowCount()); + } + result.addAll(physicalSplits); + } + } catch (IOException | IllegalArgumentException | IllegalStateException e) { + throw new UserException("Failed to plan native Paimon BLOB files: " + e.getMessage(), e); + } + return Optional.of(result); + } + + /** + * Selects the single ordinary Parquet file that supplies columns and original row positions + * for joining managed .blob files. Multiple ordinary files require column merging, which + * this native path does not support. Returns null for layouts that must fall back to JNI. + */ + private DataFileMeta findBlobParquetAnchor(List group, Set projectedBlobIds) + throws UserException { + DataFileMeta anchor = null; + for (DataFileMeta file : group) { + if (BlobFileFormat.isBlobFile(file.fileName())) { + continue; + } + // Multiple ordinary column files require a column-merge reader. Choosing one of them + // could omit newer ordinary values, even when all their row ranges match. + if (anchor != null || !"parquet".equals(file.fileFormat())) { + return null; + } + anchor = file; + } + if (anchor == null) { + return null; + } + // Descriptor-only and external-storage descriptor modes store descriptors in Parquet, + // rather than payloads in managed .blob files; projected columns in these modes use JNI. + for (DataField field : getHistoryTableSchema(anchor.schemaId()).logicalRowType().getFields()) { + if (field.type().getTypeRoot() == DataTypeRoot.BLOB + && projectedBlobIds.contains(field.id()) + && (anchor.writeCols() == null || anchor.writeCols().contains(field.name()))) { + return null; + } + } + return anchor; + } + + private TPaimonBlobFileDesc createBlobFileDescriptor(DataSplit split, DataFileMeta file) throws UserException { + DataField field = getHistoryTableSchema(file.schemaId()).logicalRowType().getField(file.writeCols().get(0)); + if (field.type().getTypeRoot() != DataTypeRoot.BLOB) { + throw new UserException("Managed Paimon BLOB file has a non-BLOB field: " + field.name()); + } + TPaimonBlobFileDesc descriptor = new TPaimonBlobFileDesc(); + descriptor.setPath(LocationPath.of(paimonDataFilePath(split, file), storagePropertiesMap) + .toStorageLocation().toString()); + descriptor.setFileSize(file.fileSize()); + descriptor.setFirstRowId(file.nonNullFirstRowId()); + descriptor.setRowCount(file.rowCount()); + descriptor.setFieldId(field.id()); + return descriptor; + } + + private static String paimonDataFilePath(DataSplit split, DataFileMeta file) { + return file.externalPath().orElse(split.bucketPath() + "/" + file.fileName()); + } + @VisibleForTesting Map getBackendPaimonOptions() { if (source == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java index 4a8808517b2176..2150c039a63bd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSplit.java @@ -21,6 +21,7 @@ import org.apache.doris.datasource.FileSplit; import org.apache.doris.datasource.SplitCreator; import org.apache.doris.datasource.TableFormatType; +import org.apache.doris.thrift.TPaimonBlobFileDesc; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.table.source.DataSplit; @@ -42,6 +43,28 @@ public class PaimonSplit extends FileSplit { private Optional optRowCount = Optional.empty(); private Optional schemaId = Optional.empty(); private Map paimonPartitionValues = null; + private List blobFiles; + // Row origin and count of the complete Parquet anchor, shared by all its byte splits. + private long anchorFirstRowId; + private long anchorRowCount; + + public void setBlobFiles(List files, long anchorFirstRowId, long anchorRowCount) { + this.blobFiles = files; + this.anchorFirstRowId = anchorFirstRowId; + this.anchorRowCount = anchorRowCount; + } + + public List getBlobFiles() { + return blobFiles; + } + + public long getAnchorFirstRowId() { + return anchorFirstRowId; + } + + public long getAnchorRowCount() { + return anchorRowCount; + } /** * Constructor for Paimon splits. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index f6eaf55e5f13c5..c5cf146d1f2fdf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -818,6 +818,8 @@ public String toString() { public static final String ENABLE_PAIMON_RUST_READER = "enable_paimon_rust_reader"; + public static final String PAIMON_BLOB_MAX_SIZE = "paimon_blob_max_size"; + public static final String ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE = "enable_count_push_down_for_external_table"; public static final String FETCH_ALL_FE_FOR_SYSTEM_TABLE = "fetch_all_fe_for_system_table"; @@ -3116,6 +3118,13 @@ public Map getForceEagerAggHintMap() { "Use paimon-rust for non-native Paimon reads, higher priority than paimon-cpp"}) private boolean enablePaimonRustReader = false; + @VariableMgr.VarAttr(name = PAIMON_BLOB_MAX_SIZE, needForward = true, + checker = "checkPaimonBlobMaxSize", + description = {"读取单个 Paimon BLOB inline 内容的最大字节数，超过限制时报错；descriptor 模式不受限制", + "Maximum inline payload size in bytes for one Paimon BLOB value. " + + "Larger values fail the query. Descriptor output is exempt."}) + private long paimonBlobMaxSize = 10L * 1024 * 1024; + @VariableMgr.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE, fuzzy = true, description = {"对外表启用 count(*) 下推优化", "enable count(*) pushdown optimization for external table"}) @@ -5835,6 +5844,7 @@ public TQueryOptions toThrift() { tResult.setEnableExprZonemapFilter(enableExprZonemapFilter); tResult.setFilePresignedUrlTtlSeconds(filePresignedUrlTtlSeconds); tResult.setEnablePaimonRustReader(enablePaimonRustReader); + tResult.setPaimonBlobMaxSize(paimonBlobMaxSize); tResult.setCheckOrcInitSargsSuccess(checkOrcInitSargsSuccess); tResult.setTruncateCharOrVarcharColumns(truncateCharOrVarcharColumns); @@ -6456,6 +6466,14 @@ public void checkBatchSize(String batchSize) { } } + public void checkPaimonBlobMaxSize(String value) { + long size = Long.parseLong(value); + if (size < 1 || size > Integer.MAX_VALUE) { + throw new InvalidParameterException(PAIMON_BLOB_MAX_SIZE + + " must be between 1 and " + Integer.MAX_VALUE + " bytes"); + } + } + public void checkLanceFragmentsPerSplit(String value) { if (Integer.parseInt(value) < 0) { diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 72952a25702d28..003036bfbde1dc 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -514,6 +514,9 @@ struct TQueryOptions { // enable_paimon_cpp_reader. 229: optional bool enable_paimon_rust_reader = false; + // Maximum payload bytes per Paimon BLOB materialized as FILE inline. Descriptors are exempt. + 230: optional i64 paimon_blob_max_size = 10485760; + // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index bad66e1aca1ae4..72aeb1b15d2574 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -360,6 +360,16 @@ enum TPaimonReaderType { PAIMON_RUST = 3, } +// A managed BLOB column file, aligned with a Parquet anchor by absolute Paimon row id. +struct TPaimonBlobFileDesc { + 1: optional string path; + 2: optional i64 file_size; + // This .blob file's own logical row range: [first_row_id, first_row_id + row_count). + 3: optional i64 first_row_id; + 4: optional i64 row_count; + 5: optional i32 field_id; +} + struct TPaimonFileDesc { 1: optional string paimon_split 2: optional string paimon_column_names @@ -385,6 +395,13 @@ struct TPaimonFileDesc { // paimon-rust: non-default branch name. Unset means main branch (matches // upstream paimon commit 742da63: null-if-DEFAULT_MAIN_BRANCH). 19: optional string paimon_branch + // Field 20 is reserved for original_file_path on master. + 21: optional list blob_files + // Logical row ID of row position 0 in the complete Parquet anchor. Byte splits of + // the same anchor share this origin; this is not the first row returned by a split. + 22: optional i64 anchor_first_row_id + // Complete anchor range: [anchor_first_row_id, anchor_first_row_id + anchor_row_count). + 23: optional i64 anchor_row_count } struct TTrinoConnectorFileDesc { diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out b/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out new file mode 100644 index 00000000000000..b631991409dc09 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out @@ -0,0 +1,274 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !show_tables -- +blob_all_null +blob_basic +blob_compacted +blob_descriptor_default +blob_empty +blob_multi_columns +blob_multi_snapshot +blob_orc_fallback +blob_partitioned +blob_rollover +blob_schema_evolution +blob_size_limit + +-- !basic_lengths -- +1 null_first true \N \N +2 empty false 0 d41d8cd98f00b204e9800998ecf8427e +3 one_zero false 1 93b885adfe0da089cdf634904fd59f71 +4 one_ascii_max false 1 83acb6e67e50e31db6ed341dd2de1595 +5 one_high_bit false 1 8d39dd7eef115ea6975446ef4082951f +6 one_ff false 1 00594fd4f42ba43fc1ca0427a0576295 +7 embedded_zero false 4 8f3d327c821490c16edebab73f2df235 +8 all_byte_values false 256 e2c865db4162bed963bfaa9ef6ac18f0 +9 utf8 false 15 78bed95a1ec30c514da54f26559fbaf5 +10 hello_world false 11 5eb63bbbe01eeed093cb22bb8f5acdc3 +11 limit_minus_one false 7 fcea920f7412b5da7be0cf42b8c93759 +12 limit_exact false 8 25d55ad283aa400af464c76d713c07ad +13 limit_plus_one false 9 25f9e794323b453885f5181f1b624d0b +14 json false 36 51f2ed1209e7b51547a3f8b3f7da4806 +15 uri_is_payload false 30 65b998ea7848368c49ff60a8d2f9c01e +16 control_bytes false 7 f75dd03e2fcb01df6fcd503fa8a2e8c0 +17 repeated_payload false 11 5eb63bbbe01eeed093cb22bb8f5acdc3 +18 null_middle true \N \N +19 empty_after_null false 0 d41d8cd98f00b204e9800998ecf8427e +20 length_1 false 1 0cc175b9c0f1b6a831c399e269772661 +21 length_47 false 47 4c2d8bcb02d982d7cb77f649c0a2dea8 +22 length_48 false 48 11f7106d9ad3167e448573f7e8652cc7 +23 length_49 false 49 ff0452236de675e6a004d3cee2c3980a +24 length_111 false 111 089f243d1e831c5879aa375ee364a06e +25 length_112 false 112 7fb848ee5850dde13e53d79c31371731 +26 length_113 false 113 4a9df97d102ac5552e89b5620101c84d +27 length_127 false 127 020406e1d05cdc2aa287641f7ae2cc39 +28 length_128 false 128 73045f89f89b1604b62a6ae1ab4d4133 +29 length_129 false 129 3024ad43cc7ea181c2c1a7c67ef0c231 +30 length_255 false 255 46bc249a5a8fc5d622cf12c42c463ae0 +31 length_256 false 256 91bfe9a0d83cfafac19af2adcffeec7a +32 length_257 false 257 a09a76e3d8193f1250c8f6adc38dffb4 +33 length_8175 false 8175 e90b828cba5eceb774af11993ccbdb50 +34 empty_after_large false 0 d41d8cd98f00b204e9800998ecf8427e +35 length_8176 false 8176 be81d8e4d2cb61fb51fe3484e45324f7 +36 null_after_large true \N \N +37 length_8177 false 8177 31c7dabd0b96e868b1350be1ed19f59b +38 small_after_large false 1 9dd4e461268c8034f5c8564e155c67a6 +39 null_last true \N \N + +-- !basic_bytes -- +3 00 +4 7F +5 80 +6 FF +7 004100FF +8 000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F808182838485868788898A8B8C8D8E8F909192939495969798999A9B9C9D9E9FA0A1A2A3A4A5A6A7A8A9AAABACADAEAFB0B1B2B3B4B5B6B7B8B9BABBBCBDBEBFC0C1C2C3C4C5C6C7C8C9CACBCCCDCECFD0D1D2D3D4D5D6D7D8D9DADBDCDDDEDFE0E1E2E3E4E5E6E7E8E9EAEBECEDEEEFF0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF +16 00090A0D1B7FFF + +-- !text_payloads -- +9 你好Doris🌍 +10 hello world +14 {"id":1,"name":"blob","active":true} +15 s3://not-a-reference/image.bin +17 hello world + +-- !basic_integrity -- +39 4 3 0 0 0 + +-- !blob_predicate -- +10 +17 + +-- !and_predicate -- +17 + +-- !or_predicate -- +1 \N +10 68656C6C6F20776F726C64 +17 68656C6C6F20776F726C64 + +-- !empty_count -- +0 + +-- !empty_projection -- + +-- !all_null -- +1 true true +2 true true +3 true true +4 true true + +-- !multi_columns -- +1 both_null \N \N \N +2 both_empty 0 d41d8cd98f00b204e9800998ecf8427e +3 right_null \N 5 5d41402abc4b2a76b9719d911017c592 +4 left_null 776F726C64 \N \N +5 different 68656C6C6F20776F726C64 2 d07d34efac6328007ad67c7e0a985e00 +6 left_empty FF0080 0 d41d8cd98f00b204e9800998ecf8427e +7 right_empty 3 60cdccd4000580a3c394b8ad6ea9b899 +8 reuse_buffer 01 257 d36a37136d406826a19f8d222217fcf2 + +-- !multi_integrity -- +8 0 0 + +-- !only_thumbnail -- +1 \N +2 +4 776F726C64 +6 FF0080 + +-- !partition_summary -- +\N 1 13 0 +a 6 21 0 +b 6 57 0 + +-- !partition_filter -- +7 004100FF +9 E4BDA0E5A5BD446F726973F09F8C8D +12 3132333435363738 + +-- !null_partition -- +13 9 + +-- !multi_snapshot -- +1 null_first \N +2 empty +7 embedded_zero 004100FF +10 duplicate_id_new_payload 6E6577 +10 hello_world 68656C6C6F20776F726C64 +12 limit_exact 3132333435363738 + +-- !compacted -- +1 \N +2 +7 004100FF +10 68656C6C6F20776F726C64 +12 3132333435363738 +13 313233343536373839 + +-- !schema_evolution -- +1 null_first \N \N \N +2 empty after-schema-change 0 +10 hello_world \N 11 68656C6C6F20776F726C64 +12 limit_exact after-schema-change 8 3132333435363738 + +-- !orc_fallback -- +1 \N +2 +7 004100FF +10 68656C6C6F20776F726C64 +12 3132333435363738 +13 313233343536373839 + +-- !rollover_integrity -- +1025 524800 0 0 + +-- !rollover_sample -- +0 \N \N \N \N +1 0 d41d8cd98f00b204e9800998ecf8427e 7 7468756D622D31 +2 12 6f0ca275b8e4b8bb624c98f1a090548a 7 7468756D622D32 +16 26 9da5284978c0b29bd7e8f7bf6cf5f8ae 8 7468756D622D3136 +17 \N \N 8 7468756D622D3137 +18 0 d41d8cd98f00b204e9800998ecf8427e 8 7468756D622D3138 +999 39 ef372a72a1d2733ee3bb5e9fcded98dd 9 7468756D622D393939 +1000 40 9d68a9f1bf1e212dc016b40c7a1df164 10 7468756D622D31303030 +1001 41 49e4505c3098e81bb126a35354867739 \N \N +1023 63 b67ce87ecb89fe2297e0abf11b4c25bc 10 7468756D622D31303233 +1024 64 c15bef66b97d734681f98710fdad44c6 10 7468756D622D31303234 +1025 65 cfd656daa46cc9dc8a8342babfc484de 10 7468756D622D31303235 +1999 69 5cdbb04f3d0fdf124408f2c96826b2b8 10 7468756D622D31393939 +2000 70 7366cff26c9c67f2377a55be15e6bd33 10 7468756D622D32303030 +2047 20 be471aa448ce549c3a0bbba561cc0e26 10 7468756D622D32303437 +2048 21 9ee6ab7f44c51de6b3f523f763e37ee3 10 7468756D622D32303438 +2049 22 731526a206779742c691f3d071228a77 10 7468756D622D32303439 +2999 99 6c7db4c1ac22a51c2fa6b3ab671cce15 10 7468756D622D32393939 +3000 100 9711bf7d004a4f16e1c37b3f4b92cde5 10 7468756D622D33303030 +3071 74 8884a53c9fc2cb67b004b65c77925812 10 7468756D622D33303731 +3072 75 47c29a38e14e2e994be773fcd9bd80ec 10 7468756D622D33303732 +3999 32 59b1cf0e396cf89e54efa94fd125197b 10 7468756D622D33393939 +4000 33 fbf1e24f4ef01a2050c2162d476fe804 10 7468756D622D34303030 +4095 31 d9277c08d8304f91a6e555bad3c250e9 \N \N +4096 32 4770b137910c7f98abcee1dc80314cb4 10 7468756D622D34303936 + +-- !rollover_sparse -- +3988 eb7c3032b71cc39376dc41f074bd31b0 d78704e16674820f22a0400d0c7e8258 +2991 7deb30a4d53c7bc0b02288357347c661 3e73e7f18a6ecefd83f0fd5d2ebfb917 +1994 2e772a63a369d4b395c642afebc086cd 52ddfab9d614c73b878fcf9fdf31a7cf + +-- !rollover_cross_predicate -- +16 7468756D622D3136 +999 7468756D622D393939 +1000 7468756D622D31303030 +1001 \N + +-- !descriptor_sample -- +1 true \N \N true true true +2 false true 0 true true true +3 false true 1 true true true +7 false true 4 true true true +8 false true 256 true true true +9 false true 15 true true true +10 false true 11 true true true +13 false true 9 true true true +33 false true 8175 true true true +34 false true 0 true true true +35 false true 8176 true true true +36 true \N \N true true true +37 false true 8177 true true true +38 false true 1 true true true +39 true \N \N true true true + +-- !descriptor_integrity -- +39 0 + +-- !rollover_descriptor_metadata -- +image 4097 241 0 +thumbnail 4097 316 0 + +-- !rollover_descriptor_files -- +image true +thumbnail true + +-- !rollover_descriptor_offsets -- +image 3856 0 +thumbnail 3781 0 + +-- !table_default_descriptor -- +1 \N true +2 0 true +10 11 true + +-- !table_override_inline -- +1 \N +2 +10 68656C6C6F20776F726C64 + +-- !table_default_again -- +1 \N true +2 0 true +10 11 true + +-- !limit_allowed -- +1 \N \N +2 0 +11 7 31323334353637 +12 8 3132333435363738 + +-- !limit_without_blob -- +13 + +-- !limit_count -- +5 + +-- !limit_only_thumbnail -- +8 01 + +-- !limit_descriptor -- +13 9 true + +-- !limit_raised -- +13 9 313233343536373839 + +-- !table_default_with_limit -- +1 \N true +2 0 true +10 11 true + diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out b/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out new file mode 100644 index 00000000000000..b631991409dc09 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out @@ -0,0 +1,274 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !show_tables -- +blob_all_null +blob_basic +blob_compacted +blob_descriptor_default +blob_empty +blob_multi_columns +blob_multi_snapshot +blob_orc_fallback +blob_partitioned +blob_rollover +blob_schema_evolution +blob_size_limit + +-- !basic_lengths -- +1 null_first true \N \N +2 empty false 0 d41d8cd98f00b204e9800998ecf8427e +3 one_zero false 1 93b885adfe0da089cdf634904fd59f71 +4 one_ascii_max false 1 83acb6e67e50e31db6ed341dd2de1595 +5 one_high_bit false 1 8d39dd7eef115ea6975446ef4082951f +6 one_ff false 1 00594fd4f42ba43fc1ca0427a0576295 +7 embedded_zero false 4 8f3d327c821490c16edebab73f2df235 +8 all_byte_values false 256 e2c865db4162bed963bfaa9ef6ac18f0 +9 utf8 false 15 78bed95a1ec30c514da54f26559fbaf5 +10 hello_world false 11 5eb63bbbe01eeed093cb22bb8f5acdc3 +11 limit_minus_one false 7 fcea920f7412b5da7be0cf42b8c93759 +12 limit_exact false 8 25d55ad283aa400af464c76d713c07ad +13 limit_plus_one false 9 25f9e794323b453885f5181f1b624d0b +14 json false 36 51f2ed1209e7b51547a3f8b3f7da4806 +15 uri_is_payload false 30 65b998ea7848368c49ff60a8d2f9c01e +16 control_bytes false 7 f75dd03e2fcb01df6fcd503fa8a2e8c0 +17 repeated_payload false 11 5eb63bbbe01eeed093cb22bb8f5acdc3 +18 null_middle true \N \N +19 empty_after_null false 0 d41d8cd98f00b204e9800998ecf8427e +20 length_1 false 1 0cc175b9c0f1b6a831c399e269772661 +21 length_47 false 47 4c2d8bcb02d982d7cb77f649c0a2dea8 +22 length_48 false 48 11f7106d9ad3167e448573f7e8652cc7 +23 length_49 false 49 ff0452236de675e6a004d3cee2c3980a +24 length_111 false 111 089f243d1e831c5879aa375ee364a06e +25 length_112 false 112 7fb848ee5850dde13e53d79c31371731 +26 length_113 false 113 4a9df97d102ac5552e89b5620101c84d +27 length_127 false 127 020406e1d05cdc2aa287641f7ae2cc39 +28 length_128 false 128 73045f89f89b1604b62a6ae1ab4d4133 +29 length_129 false 129 3024ad43cc7ea181c2c1a7c67ef0c231 +30 length_255 false 255 46bc249a5a8fc5d622cf12c42c463ae0 +31 length_256 false 256 91bfe9a0d83cfafac19af2adcffeec7a +32 length_257 false 257 a09a76e3d8193f1250c8f6adc38dffb4 +33 length_8175 false 8175 e90b828cba5eceb774af11993ccbdb50 +34 empty_after_large false 0 d41d8cd98f00b204e9800998ecf8427e +35 length_8176 false 8176 be81d8e4d2cb61fb51fe3484e45324f7 +36 null_after_large true \N \N +37 length_8177 false 8177 31c7dabd0b96e868b1350be1ed19f59b +38 small_after_large false 1 9dd4e461268c8034f5c8564e155c67a6 +39 null_last true \N \N + +-- !basic_bytes -- +3 00 +4 7F +5 80 +6 FF +7 004100FF +8 000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F808182838485868788898A8B8C8D8E8F909192939495969798999A9B9C9D9E9FA0A1A2A3A4A5A6A7A8A9AAABACADAEAFB0B1B2B3B4B5B6B7B8B9BABBBCBDBEBFC0C1C2C3C4C5C6C7C8C9CACBCCCDCECFD0D1D2D3D4D5D6D7D8D9DADBDCDDDEDFE0E1E2E3E4E5E6E7E8E9EAEBECEDEEEFF0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF +16 00090A0D1B7FFF + +-- !text_payloads -- +9 你好Doris🌍 +10 hello world +14 {"id":1,"name":"blob","active":true} +15 s3://not-a-reference/image.bin +17 hello world + +-- !basic_integrity -- +39 4 3 0 0 0 + +-- !blob_predicate -- +10 +17 + +-- !and_predicate -- +17 + +-- !or_predicate -- +1 \N +10 68656C6C6F20776F726C64 +17 68656C6C6F20776F726C64 + +-- !empty_count -- +0 + +-- !empty_projection -- + +-- !all_null -- +1 true true +2 true true +3 true true +4 true true + +-- !multi_columns -- +1 both_null \N \N \N +2 both_empty 0 d41d8cd98f00b204e9800998ecf8427e +3 right_null \N 5 5d41402abc4b2a76b9719d911017c592 +4 left_null 776F726C64 \N \N +5 different 68656C6C6F20776F726C64 2 d07d34efac6328007ad67c7e0a985e00 +6 left_empty FF0080 0 d41d8cd98f00b204e9800998ecf8427e +7 right_empty 3 60cdccd4000580a3c394b8ad6ea9b899 +8 reuse_buffer 01 257 d36a37136d406826a19f8d222217fcf2 + +-- !multi_integrity -- +8 0 0 + +-- !only_thumbnail -- +1 \N +2 +4 776F726C64 +6 FF0080 + +-- !partition_summary -- +\N 1 13 0 +a 6 21 0 +b 6 57 0 + +-- !partition_filter -- +7 004100FF +9 E4BDA0E5A5BD446F726973F09F8C8D +12 3132333435363738 + +-- !null_partition -- +13 9 + +-- !multi_snapshot -- +1 null_first \N +2 empty +7 embedded_zero 004100FF +10 duplicate_id_new_payload 6E6577 +10 hello_world 68656C6C6F20776F726C64 +12 limit_exact 3132333435363738 + +-- !compacted -- +1 \N +2 +7 004100FF +10 68656C6C6F20776F726C64 +12 3132333435363738 +13 313233343536373839 + +-- !schema_evolution -- +1 null_first \N \N \N +2 empty after-schema-change 0 +10 hello_world \N 11 68656C6C6F20776F726C64 +12 limit_exact after-schema-change 8 3132333435363738 + +-- !orc_fallback -- +1 \N +2 +7 004100FF +10 68656C6C6F20776F726C64 +12 3132333435363738 +13 313233343536373839 + +-- !rollover_integrity -- +1025 524800 0 0 + +-- !rollover_sample -- +0 \N \N \N \N +1 0 d41d8cd98f00b204e9800998ecf8427e 7 7468756D622D31 +2 12 6f0ca275b8e4b8bb624c98f1a090548a 7 7468756D622D32 +16 26 9da5284978c0b29bd7e8f7bf6cf5f8ae 8 7468756D622D3136 +17 \N \N 8 7468756D622D3137 +18 0 d41d8cd98f00b204e9800998ecf8427e 8 7468756D622D3138 +999 39 ef372a72a1d2733ee3bb5e9fcded98dd 9 7468756D622D393939 +1000 40 9d68a9f1bf1e212dc016b40c7a1df164 10 7468756D622D31303030 +1001 41 49e4505c3098e81bb126a35354867739 \N \N +1023 63 b67ce87ecb89fe2297e0abf11b4c25bc 10 7468756D622D31303233 +1024 64 c15bef66b97d734681f98710fdad44c6 10 7468756D622D31303234 +1025 65 cfd656daa46cc9dc8a8342babfc484de 10 7468756D622D31303235 +1999 69 5cdbb04f3d0fdf124408f2c96826b2b8 10 7468756D622D31393939 +2000 70 7366cff26c9c67f2377a55be15e6bd33 10 7468756D622D32303030 +2047 20 be471aa448ce549c3a0bbba561cc0e26 10 7468756D622D32303437 +2048 21 9ee6ab7f44c51de6b3f523f763e37ee3 10 7468756D622D32303438 +2049 22 731526a206779742c691f3d071228a77 10 7468756D622D32303439 +2999 99 6c7db4c1ac22a51c2fa6b3ab671cce15 10 7468756D622D32393939 +3000 100 9711bf7d004a4f16e1c37b3f4b92cde5 10 7468756D622D33303030 +3071 74 8884a53c9fc2cb67b004b65c77925812 10 7468756D622D33303731 +3072 75 47c29a38e14e2e994be773fcd9bd80ec 10 7468756D622D33303732 +3999 32 59b1cf0e396cf89e54efa94fd125197b 10 7468756D622D33393939 +4000 33 fbf1e24f4ef01a2050c2162d476fe804 10 7468756D622D34303030 +4095 31 d9277c08d8304f91a6e555bad3c250e9 \N \N +4096 32 4770b137910c7f98abcee1dc80314cb4 10 7468756D622D34303936 + +-- !rollover_sparse -- +3988 eb7c3032b71cc39376dc41f074bd31b0 d78704e16674820f22a0400d0c7e8258 +2991 7deb30a4d53c7bc0b02288357347c661 3e73e7f18a6ecefd83f0fd5d2ebfb917 +1994 2e772a63a369d4b395c642afebc086cd 52ddfab9d614c73b878fcf9fdf31a7cf + +-- !rollover_cross_predicate -- +16 7468756D622D3136 +999 7468756D622D393939 +1000 7468756D622D31303030 +1001 \N + +-- !descriptor_sample -- +1 true \N \N true true true +2 false true 0 true true true +3 false true 1 true true true +7 false true 4 true true true +8 false true 256 true true true +9 false true 15 true true true +10 false true 11 true true true +13 false true 9 true true true +33 false true 8175 true true true +34 false true 0 true true true +35 false true 8176 true true true +36 true \N \N true true true +37 false true 8177 true true true +38 false true 1 true true true +39 true \N \N true true true + +-- !descriptor_integrity -- +39 0 + +-- !rollover_descriptor_metadata -- +image 4097 241 0 +thumbnail 4097 316 0 + +-- !rollover_descriptor_files -- +image true +thumbnail true + +-- !rollover_descriptor_offsets -- +image 3856 0 +thumbnail 3781 0 + +-- !table_default_descriptor -- +1 \N true +2 0 true +10 11 true + +-- !table_override_inline -- +1 \N +2 +10 68656C6C6F20776F726C64 + +-- !table_default_again -- +1 \N true +2 0 true +10 11 true + +-- !limit_allowed -- +1 \N \N +2 0 +11 7 31323334353637 +12 8 3132333435363738 + +-- !limit_without_blob -- +13 + +-- !limit_count -- +5 + +-- !limit_only_thumbnail -- +8 01 + +-- !limit_descriptor -- +13 9 true + +-- !limit_raised -- +13 9 313233343536373839 + +-- !table_default_with_limit -- +1 \N true +2 0 true +10 11 true + diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy new file mode 100644 index 00000000000000..e9c37a9cbda752 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy @@ -0,0 +1,443 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Fixtures: iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql. +// Golden results are derived from those payloads, including binary values unsuitable for text. +suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker_doris") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_paimon_blob_jni" + try { + sql "drop catalog if exists ${catalogName}" + sql """ + create catalog ${catalogName} properties ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.region' = 'us-east-1', + 's3.path.style.access' = 'true' + ) + """ + sql "use `${catalogName}`.`test_paimon_spark_blob`" + order_qt_show_tables "show tables" + + sql "set enable_file_scanner_v2 = true" + sql "set enable_paimon_rust_reader = false" + sql "set enable_count_push_down_for_external_table = false" + sql "set batch_size = 1024" + sql "set force_jni_scanner = true" + sql "set paimon_blob_max_size = 10485760" + + ["blob_basic", "blob_all_null", "blob_size_limit", "blob_multi_columns", + "blob_partitioned", "blob_multi_snapshot", "blob_rollover", "blob_schema_evolution", + "blob_compacted", "blob_descriptor_default"].each { table -> + explain { + sql "select id, image from ${table}" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == 0 + } + } + } + + explain { + sql "select id, image from blob_basic@options('blob-as-descriptor'='true')" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == 0 + } + } + + explain { + sql "select thumbnail, id, image from blob_multi_columns" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == 0 + } + } + + explain { + sql "select thumbnail, id, image from blob_rollover" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == 0 + } + } + + explain { + sql "select thumbnail, id, image from blob_rollover@options('blob-as-descriptor'='true')" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == 0 + } + } + + // ORC anchors also use the JNI reader. + explain { + sql "select id, image from blob_orc_fallback" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == 0 + } + } + + // Inline output: NULL, empty, binary and UTF-8 payloads, with metadata children left NULL. + qt_basic_lengths """ + select id, case_name, image is null, + length(element_at(image, 'inline')), + md5(cast(element_at(image, 'inline') as string)) + from blob_basic order by id + """ + qt_basic_bytes """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_basic where id in (3, 4, 5, 6, 7, 8, 16) order by id + """ + qt_text_payloads """ + select id, cast(element_at(image, 'inline') as string) + from blob_basic where id in (9, 10, 14, 15, 17) order by id + """ + // Full-payload checks use independent Spark-written ordinary columns as the oracle. + // Inline output must leave every metadata child NULL, including size. + qt_basic_integrity """ + select count(*), sum(image is null), + sum(image is not null and length(element_at(image, 'inline')) = 0), + sum(not (length(element_at(image, 'inline')) <=> expected_size)), + sum(not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)), + sum(element_at(image, 'uri') is not null + or element_at(image, 'offset') is not null + or element_at(image, 'size') is not null + or element_at(image, 'content_type') is not null + or element_at(image, 'checksum') is not null) + from blob_basic + """ + qt_blob_predicate """ + select id from blob_basic + where cast(element_at(image, 'inline') as string) = 'hello world' order by id + """ + qt_and_predicate """ + select id from blob_basic + where id > 10 and cast(element_at(image, 'inline') as string) = 'hello world' order by id + """ + qt_or_predicate """ + select id, hex(cast(element_at(image, 'inline') as string)) from blob_basic + where id = 1 or cast(element_at(image, 'inline') as string) = 'hello world' order by id + """ + + qt_empty_count "select count(*) from blob_empty" + qt_empty_projection """ + select id, length(element_at(image, 'inline')) from blob_empty order by id + """ + qt_all_null """ + select id, image is null, element_at(image, 'inline') is null + from blob_all_null order by id + """ + // Two BLOB columns: independent values, projection order and filters on the other column. + qt_multi_columns """ + select id, note, hex(cast(element_at(thumbnail, 'inline') as string)), + length(element_at(image, 'inline')), + md5(cast(element_at(image, 'inline') as string)) + from blob_multi_columns order by id + """ + qt_multi_integrity """ + select count(*), + sum(not (length(element_at(image, 'inline')) <=> expected_size) + or not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)), + sum(not (length(element_at(thumbnail, 'inline')) <=> expected_thumbnail_size) + or not (hex(cast(element_at(thumbnail, 'inline') as string)) + <=> expected_thumbnail_hex)) + from blob_multi_columns + """ + // Filter on one BLOB but output only the other, including NULL and empty values. + qt_only_thumbnail """ + select id, hex(cast(element_at(thumbnail, 'inline') as string)) + from blob_multi_columns + where image is null or length(element_at(image, 'inline')) = 0 order by id + """ + // Partition pruning, later snapshots, compaction and schema changes must preserve row mapping. + qt_partition_summary """ + select region, count(*), sum(id), + sum(not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)) + from blob_partitioned group by region order by region nulls first + """ + qt_partition_filter """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_partitioned where region = 'b' and id in (7, 9, 12) order by id + """ + qt_null_partition """ + select id, length(element_at(image, 'inline')) + from blob_partitioned where region is null order by id + """ + qt_multi_snapshot """ + select id, case_name, hex(cast(element_at(image, 'inline') as string)) + from blob_multi_snapshot order by id, case_name + """ + qt_compacted """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_compacted order by id + """ + qt_schema_evolution """ + select id, label, extra, length(element_at(image, 'inline')), + hex(cast(element_at(image, 'inline') as string)) + from blob_schema_evolution order by id + """ + qt_orc_fallback """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_orc_fallback order by id + """ + + // Read 1025 rows to cross the 1024-row batch boundary, checking both BLOB lengths. + // MD5/HEX samples below check bytes; descriptor cases check rollover across all files. + qt_rollover_integrity """ + select count(*) as row_count, sum(id) as id_sum, + sum(not (length(element_at(image, 'inline')) <=> expected_size)) + as invalid_image_length_count, + sum(not (length(element_at(thumbnail, 'inline')) <=> expected_thumbnail_size)) + as invalid_thumbnail_length_count + from blob_rollover + where id between 0 and 1024 + """ + qt_rollover_sample """ + select id, length(element_at(image, 'inline')), + md5(cast(element_at(image, 'inline') as string)), + length(element_at(thumbnail, 'inline')), + hex(cast(element_at(thumbnail, 'inline') as string)) + from blob_rollover + where id in (0, 1, 2, 16, 17, 18, 999, 1000, 1001, 1023, 1024, 1025, + 1999, 2000, 2047, 2048, 2049, 2999, 3000, 3071, 3072, + 3999, 4000, 4095, 4096) + order by id + """ + // Explicit sparse IDs let Paimon apply the filter; keep descending order and LIMIT coverage. + qt_rollover_sparse """ + select id, md5(cast(element_at(image, 'inline') as string)), + md5(cast(element_at(thumbnail, 'inline') as string)) + from blob_rollover + where id in (0, 997, 1994, 2991, 3988) + order by id desc limit 3 + """ + qt_rollover_cross_predicate """ + select id, hex(cast(element_at(thumbnail, 'inline') as string)) + from blob_rollover + where id in (16, 17, 18, 999, 1000, 1001) + and cast(element_at(image, 'inline') as string) like 'row-%' + order by id + """ + + // Descriptor output returns URI/offset/size while inline remains NULL. + qt_descriptor_sample """ + select id, image is null, + case when image is null then null + else right(element_at(image, 'uri'), 5) = '.blob' + and element_at(image, 'offset') >= 4 end, + element_at(image, 'size'), element_at(image, 'inline') is null, + element_at(image, 'content_type') is null, element_at(image, 'checksum') is null + from blob_basic@options('blob-as-descriptor'='true') + where id in (1, 2, 3, 7, 8, 9, 10, 13, 33, 34, 35, 36, 37, 38, 39) + order by id + """ + qt_descriptor_integrity """ + select count(*), + sum(not (element_at(image, 'size') <=> expected_size) + or not ((element_at(image, 'uri') is null) <=> (expected_size is null)) + or not ((element_at(image, 'offset') is null) <=> (expected_size is null)) + or element_at(image, 'inline') is not null + or element_at(image, 'content_type') is not null + or element_at(image, 'checksum') is not null) + from blob_basic@options('blob-as-descriptor'='true') + """ + + // Each BLOB column has 4097 rows. Metadata errors must be zero, including for NULLs. + qt_rollover_descriptor_metadata """ + with blob_descriptors as ( + select 'image' as blob_column_name, image is null as is_null_blob, + element_at(image, 'uri') as blob_uri, + element_at(image, 'offset') as payload_offset, + element_at(image, 'size') as payload_size, + expected_size as expected_payload_size, + element_at(image, 'inline') is not null as has_inline, + element_at(image, 'content_type') is not null as has_content_type, + element_at(image, 'checksum') is not null as has_checksum + from blob_rollover@options('blob-as-descriptor'='true') + union all + select 'thumbnail', thumbnail is null, + element_at(thumbnail, 'uri'), element_at(thumbnail, 'offset'), + element_at(thumbnail, 'size'), expected_thumbnail_size, + element_at(thumbnail, 'inline') is not null, + element_at(thumbnail, 'content_type') is not null, + element_at(thumbnail, 'checksum') is not null + from blob_rollover@options('blob-as-descriptor'='true') + ) + select blob_column_name, count(*) as row_count, sum(is_null_blob) as null_count, + sum(not (is_null_blob <=> (expected_payload_size is null)) + or not (payload_size <=> expected_payload_size) + or not ((blob_uri is null) <=> is_null_blob) + or not ((payload_offset is null) <=> is_null_blob) + or (blob_uri is not null and right(blob_uri, 5) != '.blob') + or (payload_offset is not null and payload_offset < 4) + or has_inline or has_content_type or has_checksum) as invalid_metadata_count + from blob_descriptors + group by blob_column_name order by blob_column_name + """ + + // Both BLOB columns must span multiple files. Random file names and counts are not golden values. + qt_rollover_descriptor_files """ + select 'image' as blob_column_name, + count(distinct element_at(image, 'uri')) > 1 as has_multiple_blob_files + from blob_rollover@options('blob-as-descriptor'='true') + union all + select 'thumbnail', count(distinct element_at(thumbnail, 'uri')) > 1 + from blob_rollover@options('blob-as-descriptor'='true') + order by blob_column_name + """ + + // Within each file: first payload starts at 4; the next starts at offset + size + 16. + // NULL BLOBs have no record; empty payloads still consume the 16 framing bytes. + qt_rollover_descriptor_offsets """ + with blob_ranges as ( + select 'image' as blob_column_name, id, + element_at(image, 'uri') as blob_uri, + element_at(image, 'offset') as payload_offset, + element_at(image, 'size') as payload_size + from blob_rollover@options('blob-as-descriptor'='true') + where image is not null + union all + select 'thumbnail', id, element_at(thumbnail, 'uri'), + element_at(thumbnail, 'offset'), element_at(thumbnail, 'size') + from blob_rollover@options('blob-as-descriptor'='true') + where thumbnail is not null + ), expected_offsets as ( + select blob_column_name, payload_offset, + lag(payload_offset + payload_size + 16, 1, 4) over ( + partition by blob_column_name, blob_uri order by payload_offset, id + ) as expected_payload_offset + from blob_ranges + ) + select blob_column_name, count(*) as payload_count, + sum(not (payload_offset <=> expected_payload_offset)) as invalid_offset_count + from expected_offsets + group by blob_column_name order by blob_column_name + """ + + // The SDK reader's internal descriptor option must not change the user's output + // mode or poison the cached table when queries alternate between representations. + qt_table_default_descriptor """ + select id, element_at(image, 'size'), element_at(image, 'inline') is null + from blob_descriptor_default order by id + """ + qt_table_override_inline """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_descriptor_default@options('blob-as-descriptor'='false') order by id + """ + qt_table_default_again """ + select id, element_at(image, 'size'), element_at(image, 'inline') is null + from blob_descriptor_default order by id + """ + + // Small payloads exercise the limit: do not allocate 10 MiB just to test an error. + sql "set paimon_blob_max_size = 8" + qt_limit_allowed """ + select id, length(element_at(image, 'inline')), + hex(cast(element_at(image, 'inline') as string)) + from blob_size_limit where id in (1, 2, 11, 12) order by id + """ + test { + sql "select cast(element_at(image, 'inline') as string) from blob_size_limit where id = 13" + exception "paimon_blob_max_size=8" + } + test { + // BLOB-dependent filters run after materialization, so they cannot bypass the limit. + sql "select id from blob_size_limit where length(element_at(image, 'inline')) <= 8" + exception "paimon_blob_max_size=8" + } + qt_limit_without_blob "select id from blob_size_limit where id = 13" + qt_limit_count "select count(*) from blob_size_limit" + qt_limit_only_thumbnail """ + select id, hex(cast(element_at(thumbnail, 'inline') as string)) + from blob_multi_columns where id = 8 + """ + test { + sql "select cast(element_at(image, 'inline') as string) from blob_multi_columns where id = 8" + exception "paimon_blob_max_size=8" + } + qt_limit_descriptor """ + select id, element_at(image, 'size'), element_at(image, 'inline') is null + from blob_size_limit@options('blob-as-descriptor'='true') where id = 13 + """ + sql "set paimon_blob_max_size = 9" + qt_limit_raised """ + select id, length(element_at(image, 'inline')), + hex(cast(element_at(image, 'inline') as string)) + from blob_size_limit where id = 13 + """ + sql "set paimon_blob_max_size = 1" + qt_table_default_with_limit """ + select id, element_at(image, 'size'), element_at(image, 'inline') is null + from blob_descriptor_default order by id + """ + test { + sql """ + select cast(element_at(image, 'inline') as string) + from blob_descriptor_default@options('blob-as-descriptor'='false') where id = 10 + """ + exception "paimon_blob_max_size=1" + } + + // BLOB reading requires FileScannerV2 on both native and JNI paths. + sql "set enable_file_scanner_v2 = false" + test { + sql "select image from blob_basic where id = 10" + exception "Paimon BLOB columns require enable_file_scanner_v2=true" + } + sql "set enable_file_scanner_v2 = true" + + ["0", "-1", "2147483648"].each { invalidSize -> + test { + sql "set paimon_blob_max_size = ${invalidSize}" + exception "paimon_blob_max_size must be between 1 and 2147483647 bytes" + } + } + } finally { + sql "drop catalog if exists ${catalogName}" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy new file mode 100644 index 00000000000000..258ec7f7aaf8b3 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy @@ -0,0 +1,443 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Fixtures: iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql. +// Golden results are derived from those payloads, including binary values unsuitable for text. +suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_docker_doris") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_paimon_blob_native" + try { + sql "drop catalog if exists ${catalogName}" + sql """ + create catalog ${catalogName} properties ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.region' = 'us-east-1', + 's3.path.style.access' = 'true' + ) + """ + sql "use `${catalogName}`.`test_paimon_spark_blob`" + order_qt_show_tables "show tables" + + sql "set enable_file_scanner_v2 = true" + sql "set enable_paimon_rust_reader = false" + sql "set enable_count_push_down_for_external_table = false" + sql "set batch_size = 1024" + sql "set force_jni_scanner = false" + sql "set paimon_blob_max_size = 10485760" + + ["blob_basic", "blob_all_null", "blob_size_limit", "blob_multi_columns", + "blob_partitioned", "blob_multi_snapshot", "blob_rollover", "blob_schema_evolution", + "blob_compacted", "blob_descriptor_default"].each { table -> + explain { + sql "select id, image from ${table}" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == totalSplits + } + } + } + + explain { + sql "select id, image from blob_basic@options('blob-as-descriptor'='true')" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == totalSplits + } + } + + explain { + sql "select thumbnail, id, image from blob_multi_columns" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == totalSplits + } + } + + explain { + sql "select thumbnail, id, image from blob_rollover" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == totalSplits + } + } + + explain { + sql "select thumbnail, id, image from blob_rollover@options('blob-as-descriptor'='true')" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == totalSplits + } + } + + // The current native BLOB implementation needs a Parquet ordinary-column anchor. + explain { + sql "select id, image from blob_orc_fallback" + check { plan -> + def matcher = plan =~ /paimonNativeReadSplits=(\d+)\/(\d+)/ + assertTrue(matcher.find(), "Missing Paimon split counters: ${plan}") + int nativeSplits = matcher.group(1).toInteger() + int totalSplits = matcher.group(2).toInteger() + return totalSplits > 0 && nativeSplits == 0 + } + } + + // Inline output: NULL, empty, binary and UTF-8 payloads, with metadata children left NULL. + qt_basic_lengths """ + select id, case_name, image is null, + length(element_at(image, 'inline')), + md5(cast(element_at(image, 'inline') as string)) + from blob_basic order by id + """ + qt_basic_bytes """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_basic where id in (3, 4, 5, 6, 7, 8, 16) order by id + """ + qt_text_payloads """ + select id, cast(element_at(image, 'inline') as string) + from blob_basic where id in (9, 10, 14, 15, 17) order by id + """ + // Full-payload checks use independent Spark-written ordinary columns as the oracle. + // Inline output must leave every metadata child NULL, including size. + qt_basic_integrity """ + select count(*), sum(image is null), + sum(image is not null and length(element_at(image, 'inline')) = 0), + sum(not (length(element_at(image, 'inline')) <=> expected_size)), + sum(not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)), + sum(element_at(image, 'uri') is not null + or element_at(image, 'offset') is not null + or element_at(image, 'size') is not null + or element_at(image, 'content_type') is not null + or element_at(image, 'checksum') is not null) + from blob_basic + """ + qt_blob_predicate """ + select id from blob_basic + where cast(element_at(image, 'inline') as string) = 'hello world' order by id + """ + qt_and_predicate """ + select id from blob_basic + where id > 10 and cast(element_at(image, 'inline') as string) = 'hello world' order by id + """ + qt_or_predicate """ + select id, hex(cast(element_at(image, 'inline') as string)) from blob_basic + where id = 1 or cast(element_at(image, 'inline') as string) = 'hello world' order by id + """ + + qt_empty_count "select count(*) from blob_empty" + qt_empty_projection """ + select id, length(element_at(image, 'inline')) from blob_empty order by id + """ + qt_all_null """ + select id, image is null, element_at(image, 'inline') is null + from blob_all_null order by id + """ + // Two BLOB columns: independent values, projection order and filters on the other column. + qt_multi_columns """ + select id, note, hex(cast(element_at(thumbnail, 'inline') as string)), + length(element_at(image, 'inline')), + md5(cast(element_at(image, 'inline') as string)) + from blob_multi_columns order by id + """ + qt_multi_integrity """ + select count(*), + sum(not (length(element_at(image, 'inline')) <=> expected_size) + or not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)), + sum(not (length(element_at(thumbnail, 'inline')) <=> expected_thumbnail_size) + or not (hex(cast(element_at(thumbnail, 'inline') as string)) + <=> expected_thumbnail_hex)) + from blob_multi_columns + """ + // Filter on one BLOB but output only the other, including NULL and empty values. + qt_only_thumbnail """ + select id, hex(cast(element_at(thumbnail, 'inline') as string)) + from blob_multi_columns + where image is null or length(element_at(image, 'inline')) = 0 order by id + """ + // Partition pruning, later snapshots, compaction and schema changes must preserve row mapping. + qt_partition_summary """ + select region, count(*), sum(id), + sum(not (hex(cast(element_at(image, 'inline') as string)) <=> expected_hex)) + from blob_partitioned group by region order by region nulls first + """ + qt_partition_filter """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_partitioned where region = 'b' and id in (7, 9, 12) order by id + """ + qt_null_partition """ + select id, length(element_at(image, 'inline')) + from blob_partitioned where region is null order by id + """ + qt_multi_snapshot """ + select id, case_name, hex(cast(element_at(image, 'inline') as string)) + from blob_multi_snapshot order by id, case_name + """ + qt_compacted """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_compacted order by id + """ + qt_schema_evolution """ + select id, label, extra, length(element_at(image, 'inline')), + hex(cast(element_at(image, 'inline') as string)) + from blob_schema_evolution order by id + """ + qt_orc_fallback """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_orc_fallback order by id + """ + + // Read 1025 rows to cross the 1024-row batch boundary, checking both BLOB lengths. + // MD5/HEX samples below check bytes; descriptor cases check rollover across all files. + qt_rollover_integrity """ + select count(*) as row_count, sum(id) as id_sum, + sum(not (length(element_at(image, 'inline')) <=> expected_size)) + as invalid_image_length_count, + sum(not (length(element_at(thumbnail, 'inline')) <=> expected_thumbnail_size)) + as invalid_thumbnail_length_count + from blob_rollover + where id between 0 and 1024 + """ + qt_rollover_sample """ + select id, length(element_at(image, 'inline')), + md5(cast(element_at(image, 'inline') as string)), + length(element_at(thumbnail, 'inline')), + hex(cast(element_at(thumbnail, 'inline') as string)) + from blob_rollover + where id in (0, 1, 2, 16, 17, 18, 999, 1000, 1001, 1023, 1024, 1025, + 1999, 2000, 2047, 2048, 2049, 2999, 3000, 3071, 3072, + 3999, 4000, 4095, 4096) + order by id + """ + // Explicit sparse IDs let Paimon apply the filter; keep descending order and LIMIT coverage. + qt_rollover_sparse """ + select id, md5(cast(element_at(image, 'inline') as string)), + md5(cast(element_at(thumbnail, 'inline') as string)) + from blob_rollover + where id in (0, 997, 1994, 2991, 3988) + order by id desc limit 3 + """ + qt_rollover_cross_predicate """ + select id, hex(cast(element_at(thumbnail, 'inline') as string)) + from blob_rollover + where id in (16, 17, 18, 999, 1000, 1001) + and cast(element_at(image, 'inline') as string) like 'row-%' + order by id + """ + + // Descriptor output returns URI/offset/size while inline remains NULL. + qt_descriptor_sample """ + select id, image is null, + case when image is null then null + else right(element_at(image, 'uri'), 5) = '.blob' + and element_at(image, 'offset') >= 4 end, + element_at(image, 'size'), element_at(image, 'inline') is null, + element_at(image, 'content_type') is null, element_at(image, 'checksum') is null + from blob_basic@options('blob-as-descriptor'='true') + where id in (1, 2, 3, 7, 8, 9, 10, 13, 33, 34, 35, 36, 37, 38, 39) + order by id + """ + qt_descriptor_integrity """ + select count(*), + sum(not (element_at(image, 'size') <=> expected_size) + or not ((element_at(image, 'uri') is null) <=> (expected_size is null)) + or not ((element_at(image, 'offset') is null) <=> (expected_size is null)) + or element_at(image, 'inline') is not null + or element_at(image, 'content_type') is not null + or element_at(image, 'checksum') is not null) + from blob_basic@options('blob-as-descriptor'='true') + """ + + // Each BLOB column has 4097 rows. Metadata errors must be zero, including for NULLs. + qt_rollover_descriptor_metadata """ + with blob_descriptors as ( + select 'image' as blob_column_name, image is null as is_null_blob, + element_at(image, 'uri') as blob_uri, + element_at(image, 'offset') as payload_offset, + element_at(image, 'size') as payload_size, + expected_size as expected_payload_size, + element_at(image, 'inline') is not null as has_inline, + element_at(image, 'content_type') is not null as has_content_type, + element_at(image, 'checksum') is not null as has_checksum + from blob_rollover@options('blob-as-descriptor'='true') + union all + select 'thumbnail', thumbnail is null, + element_at(thumbnail, 'uri'), element_at(thumbnail, 'offset'), + element_at(thumbnail, 'size'), expected_thumbnail_size, + element_at(thumbnail, 'inline') is not null, + element_at(thumbnail, 'content_type') is not null, + element_at(thumbnail, 'checksum') is not null + from blob_rollover@options('blob-as-descriptor'='true') + ) + select blob_column_name, count(*) as row_count, sum(is_null_blob) as null_count, + sum(not (is_null_blob <=> (expected_payload_size is null)) + or not (payload_size <=> expected_payload_size) + or not ((blob_uri is null) <=> is_null_blob) + or not ((payload_offset is null) <=> is_null_blob) + or (blob_uri is not null and right(blob_uri, 5) != '.blob') + or (payload_offset is not null and payload_offset < 4) + or has_inline or has_content_type or has_checksum) as invalid_metadata_count + from blob_descriptors + group by blob_column_name order by blob_column_name + """ + + // Both BLOB columns must span multiple files. Random file names and counts are not golden values. + qt_rollover_descriptor_files """ + select 'image' as blob_column_name, + count(distinct element_at(image, 'uri')) > 1 as has_multiple_blob_files + from blob_rollover@options('blob-as-descriptor'='true') + union all + select 'thumbnail', count(distinct element_at(thumbnail, 'uri')) > 1 + from blob_rollover@options('blob-as-descriptor'='true') + order by blob_column_name + """ + + // Within each file: first payload starts at 4; the next starts at offset + size + 16. + // NULL BLOBs have no record; empty payloads still consume the 16 framing bytes. + qt_rollover_descriptor_offsets """ + with blob_ranges as ( + select 'image' as blob_column_name, id, + element_at(image, 'uri') as blob_uri, + element_at(image, 'offset') as payload_offset, + element_at(image, 'size') as payload_size + from blob_rollover@options('blob-as-descriptor'='true') + where image is not null + union all + select 'thumbnail', id, element_at(thumbnail, 'uri'), + element_at(thumbnail, 'offset'), element_at(thumbnail, 'size') + from blob_rollover@options('blob-as-descriptor'='true') + where thumbnail is not null + ), expected_offsets as ( + select blob_column_name, payload_offset, + lag(payload_offset + payload_size + 16, 1, 4) over ( + partition by blob_column_name, blob_uri order by payload_offset, id + ) as expected_payload_offset + from blob_ranges + ) + select blob_column_name, count(*) as payload_count, + sum(not (payload_offset <=> expected_payload_offset)) as invalid_offset_count + from expected_offsets + group by blob_column_name order by blob_column_name + """ + + // Query-level overrides must not change the table's default descriptor output + // when queries alternate between representations. + qt_table_default_descriptor """ + select id, element_at(image, 'size'), element_at(image, 'inline') is null + from blob_descriptor_default order by id + """ + qt_table_override_inline """ + select id, hex(cast(element_at(image, 'inline') as string)) + from blob_descriptor_default@options('blob-as-descriptor'='false') order by id + """ + qt_table_default_again """ + select id, element_at(image, 'size'), element_at(image, 'inline') is null + from blob_descriptor_default order by id + """ + + // Small payloads exercise the limit: do not allocate 10 MiB just to test an error. + sql "set paimon_blob_max_size = 8" + qt_limit_allowed """ + select id, length(element_at(image, 'inline')), + hex(cast(element_at(image, 'inline') as string)) + from blob_size_limit where id in (1, 2, 11, 12) order by id + """ + test { + sql "select cast(element_at(image, 'inline') as string) from blob_size_limit where id = 13" + exception "paimon_blob_max_size=8" + } + test { + // BLOB-dependent filters run after materialization, so they cannot bypass the limit. + sql "select id from blob_size_limit where length(element_at(image, 'inline')) <= 8" + exception "paimon_blob_max_size=8" + } + qt_limit_without_blob "select id from blob_size_limit where id = 13" + qt_limit_count "select count(*) from blob_size_limit" + qt_limit_only_thumbnail """ + select id, hex(cast(element_at(thumbnail, 'inline') as string)) + from blob_multi_columns where id = 8 + """ + test { + sql "select cast(element_at(image, 'inline') as string) from blob_multi_columns where id = 8" + exception "paimon_blob_max_size=8" + } + qt_limit_descriptor """ + select id, element_at(image, 'size'), element_at(image, 'inline') is null + from blob_size_limit@options('blob-as-descriptor'='true') where id = 13 + """ + sql "set paimon_blob_max_size = 9" + qt_limit_raised """ + select id, length(element_at(image, 'inline')), + hex(cast(element_at(image, 'inline') as string)) + from blob_size_limit where id = 13 + """ + sql "set paimon_blob_max_size = 1" + qt_table_default_with_limit """ + select id, element_at(image, 'size'), element_at(image, 'inline') is null + from blob_descriptor_default order by id + """ + test { + sql """ + select cast(element_at(image, 'inline') as string) + from blob_descriptor_default@options('blob-as-descriptor'='false') where id = 10 + """ + exception "paimon_blob_max_size=1" + } + + // BLOB reading requires FileScannerV2 on both native and JNI paths. + sql "set enable_file_scanner_v2 = false" + test { + sql "select image from blob_basic where id = 10" + exception "Paimon BLOB columns require enable_file_scanner_v2=true" + } + sql "set enable_file_scanner_v2 = true" + + ["0", "-1", "2147483648"].each { invalidSize -> + test { + sql "set paimon_blob_max_size = ${invalidSize}" + exception "paimon_blob_max_size must be between 1 and 2147483647 bytes" + } + } + } finally { + sql "drop catalog if exists ${catalogName}" + } +} From 03061906e5285270c3572213abd543f82a176438 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Fri, 16 Oct 2026 17:14:10 +0800 Subject: [PATCH 4/5] update file types --- be/src/format/jni/jni_data_bridge.cpp | 34 +-- be/src/format/jni/jni_data_bridge.h | 3 +- be/src/format_v2/jni/paimon_jni_reader.cpp | 7 +- be/test/format/jni/jni_data_bridge_test.cpp | 82 +++++++ .../format_v2/jni/paimon_jni_reader_test.cpp | 27 +++ .../doris/common/jni/vec/ColumnType.java | 46 ++-- .../doris/common/jni/vec/ColumnValue.java | 4 + .../doris/common/jni/vec/FileValue.java | 74 +++++++ .../doris/common/jni/vec/VectorColumn.java | 144 +++++++++++- .../common/jni/vec/VectorColumnFileTest.java | 207 ++++++++++++++++++ .../doris/paimon/PaimonColumnValue.java | 60 +---- .../paimon/PaimonReadTypeProjection.java | 4 +- .../apache/doris/paimon/PaimonTypeUtils.java | 10 +- .../doris/paimon/PaimonColumnValueTest.java | 97 ++++++++ .../paimon/PaimonReadTypeProjectionTest.java | 13 ++ .../paimon/PaimonReaderOptionsTest.java | 15 ++ gensrc/thrift/PaloInternalService.thrift | 3 +- 17 files changed, 718 insertions(+), 112 deletions(-) create mode 100644 fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/FileValue.java create mode 100644 fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnFileTest.java diff --git a/be/src/format/jni/jni_data_bridge.cpp b/be/src/format/jni/jni_data_bridge.cpp index 871016d9dd9567..e6b9dea9277e0a 100644 --- a/be/src/format/jni/jni_data_bridge.cpp +++ b/be/src/format/jni/jni_data_bridge.cpp @@ -46,34 +46,6 @@ namespace doris { -namespace { - -// FILE keeps its Doris type in the destination block. The JNI wire layout is the same -// six-child layout as STRUCT, so Java can reuse VectorColumn's existing nested transport. -std::string jni_file_type(const DataTypePtr& type, bool encoded_names) { - const auto& file_type = assert_cast(*remove_nullable(type)); - std::ostringstream buffer; - buffer << "struct<"; - for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) { - if (i != 0) { - buffer << ","; - } - if (encoded_names) { - std::string encoded_name; - base64_encode(file_type.get_element_name(i), &encoded_name); - buffer << "$" << encoded_name; - } else { - buffer << file_type.get_element_name(i); - } - // Type lengths remain authoritative on BE; Java only needs the binary value layout. - buffer << ":" << JniDataBridge::get_jni_type(file_type.get_element(i)); - } - buffer << ">"; - return buffer.str(); -} - -} // namespace - #define FOR_FIXED_LENGTH_TYPES(M) \ M(PrimitiveType::TYPE_TINYINT, ColumnInt8, Int8) \ M(PrimitiveType::TYPE_BOOLEAN, ColumnUInt8, UInt8) \ @@ -337,7 +309,7 @@ std::string JniDataBridge::get_jni_type(const DataTypePtr& data_type) { std::ostringstream buffer; switch (type->get_primitive_type()) { case TYPE_FILE: - return jni_file_type(data_type, false); + return "file"; case TYPE_BOOLEAN: return "boolean"; case TYPE_TINYINT: @@ -445,7 +417,7 @@ std::string JniDataBridge::get_jni_type_with_different_string(const DataTypePtr& std::ostringstream buffer; switch (data_type->get_primitive_type()) { case TYPE_FILE: - return jni_file_type(data_type, false); + return "file"; case TYPE_BOOLEAN: return "boolean"; case TYPE_TINYINT: @@ -579,7 +551,7 @@ std::string JniDataBridge::encode_string_list(const std::vector& va std::string JniDataBridge::get_jni_type_with_encoded_struct_fields(const DataTypePtr& data_type) { switch (data_type->get_primitive_type()) { case TYPE_FILE: - return jni_file_type(data_type, true); + return "file"; case TYPE_STRUCT: { const auto* type_struct = assert_cast(remove_nullable(data_type).get()); diff --git a/be/src/format/jni/jni_data_bridge.h b/be/src/format/jni/jni_data_bridge.h index 339dd600f07214..d483199cc9aef5 100644 --- a/be/src/format/jni/jni_data_bridge.h +++ b/be/src/format/jni/jni_data_bridge.h @@ -130,7 +130,8 @@ class JniDataBridge { /** * Convert a Doris DataType to its JNI type string representation. - * e.g., TYPE_INT -> "int", TYPE_DECIMAL128I -> "decimal128(p,s)" + * e.g., TYPE_INT -> "int", TYPE_DECIMAL128I -> "decimal128(p,s)", TYPE_FILE -> "file". + * FILE retains its logical type; both sides use the canonical six-child physical layout. */ static std::string get_jni_type(const DataTypePtr& data_type); diff --git a/be/src/format_v2/jni/paimon_jni_reader.cpp b/be/src/format_v2/jni/paimon_jni_reader.cpp index fd53b0b440d185..c055ecb043a395 100644 --- a/be/src/format_v2/jni/paimon_jni_reader.cpp +++ b/be/src/format_v2/jni/paimon_jni_reader.cpp @@ -88,7 +88,6 @@ std::string PaimonJniReader::connector_class() const { Status PaimonJniReader::build_scanner_params(std::map* params) const { DORIS_CHECK(params != nullptr); DORIS_CHECK(_scan_params != nullptr); - DORIS_CHECK(_runtime_state != nullptr); params->clear(); const auto& paimon_params = _current_range.table_format_params.paimon_params; @@ -103,8 +102,9 @@ Status PaimonJniReader::build_scanner_params(std::map* !_scan_params->serialized_table_cache_key.empty() ? _scan_params->serialized_table_cache_key : generate_uuid_string(); - (*params)["paimon_blob_max_size"] = - std::to_string(_runtime_state->query_options().paimon_blob_max_size); + (*params)["paimon_blob_max_size"] = std::to_string( + _runtime_state != nullptr ? _runtime_state->query_options().paimon_blob_max_size + : TQueryOptions().paimon_blob_max_size); if (_scan_params->__isset.paimon_options && !_scan_params->paimon_options.empty()) { for (const auto& kv : _scan_params->paimon_options) { @@ -123,6 +123,7 @@ Status PaimonJniReader::build_scanner_params(std::map* auto enable_io_manager_it = params->find(enable_io_manager_key); if (enable_io_manager_it != params->end() && iequal(enable_io_manager_it->second, "true") && params->find(io_manager_tmp_dir_key) == params->end()) { + DORIS_CHECK(_runtime_state != nullptr); // Keep Format V2 consistent with the legacy JNI path. Paimon creates and // removes its own paimon-* child directory under these Doris storage-root scoped // parent directories. diff --git a/be/test/format/jni/jni_data_bridge_test.cpp b/be/test/format/jni/jni_data_bridge_test.cpp index d6418c0b97e28d..08fbcf370b5200 100644 --- a/be/test/format/jni/jni_data_bridge_test.cpp +++ b/be/test/format/jni/jni_data_bridge_test.cpp @@ -22,9 +22,16 @@ #include #include +#include "core/column/column_file.h" #include "core/column/column_nullable.h" +#include "core/column/column_varbinary.h" #include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_file.h" +#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_variant_v2.h" namespace doris { @@ -35,6 +42,81 @@ long address_of(T* pointer) { return reinterpret_cast(pointer); } +TEST(JniDataBridgeTest, FileTypePreservesLogicalJniNameInNestedTypes) { + DataTypePtr file_type = std::make_shared(); + for (const auto& type : {file_type, make_nullable(file_type)}) { + EXPECT_EQ(JniDataBridge::get_jni_type(type), "file"); + EXPECT_EQ(JniDataBridge::get_jni_type_with_different_string(type), "file"); + EXPECT_EQ(JniDataBridge::get_jni_type_with_encoded_struct_fields(type), "file"); + } + auto array = std::make_shared(make_nullable(file_type)); + auto map = std::make_shared(std::make_shared(), + make_nullable(file_type)); + auto nested = std::make_shared(DataTypes {make_nullable(file_type)}, + Strings {"image"}); + EXPECT_EQ(JniDataBridge::get_jni_type(array), "array"); + EXPECT_EQ(JniDataBridge::get_jni_type(map), "map"); + EXPECT_EQ(JniDataBridge::get_jni_type(nested), "struct"); + EXPECT_EQ(JniDataBridge::get_jni_type_with_encoded_struct_fields(nested), + "struct<$aW1hZ2U=:file>"); +} + +TEST(JniDataBridgeTest, FillFileFromCanonicalJavaChildren) { + // Rows: inline payload, descriptor, SQL NULL. All six child columns must + // retain the same row count, including null placeholders. + std::array file_nulls {false, false, true}; + std::array descriptor_nulls {true, false, true}; + std::array inline_nulls {false, true, true}; + std::array all_nulls {true, true, true}; + std::string uri = "file:/data.blob"; + std::array uri_offsets {0, static_cast(uri.size()), + static_cast(uri.size())}; + std::array blob_offsets {0, 17, 0}; + std::array blob_sizes {0, 11, 0}; + std::array empty_offsets {0, 0, 0}; + std::string payload = "hello world"; + // Java -> BE VARBINARY entries carry length and pointer, not C++ StringView. + std::array inline_values { + static_cast(payload.size()), address_of(payload.data()), 0, 0, 0, 0}; + std::array meta { + address_of(file_nulls.data()), + address_of(descriptor_nulls.data()), + address_of(uri_offsets.data()), + address_of(uri.data()), + address_of(descriptor_nulls.data()), + address_of(blob_offsets.data()), + address_of(descriptor_nulls.data()), + address_of(blob_sizes.data()), + address_of(all_nulls.data()), + address_of(empty_offsets.data()), + address_of(uri.data()), + address_of(all_nulls.data()), + address_of(empty_offsets.data()), + address_of(uri.data()), + address_of(inline_nulls.data()), + address_of(inline_values.data()), + }; + JniDataBridge::TableMetaAddress address(address_of(meta.data())); + DataTypePtr type = make_nullable(std::make_shared()); + ColumnPtr column = type->create_column(); + ASSERT_TRUE(JniDataBridge::fill_column(address, column, type, 3).ok()); + const auto& nullable = assert_cast(*column); + EXPECT_EQ(nullable.get_null_map_data(), NullMap({0, 0, 1})); + const auto& file = assert_cast(nullable.get_nested_column()); + ASSERT_EQ(file.size(), 3); + for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) { + EXPECT_EQ(file.get_column(i).size(), 3); + } + const auto& uri_column = + assert_cast(file.get_column(DataTypeFile::URI_FIELD_INDEX)); + EXPECT_EQ(uri_column.get_null_map_data(), NullMap({1, 0, 1})); + EXPECT_EQ(uri_column.get_nested_column().get_data_at(1).to_string(), uri); + const auto& inline_column = + assert_cast(file.get_column(DataTypeFile::INLINE_FIELD_INDEX)); + EXPECT_EQ(inline_column.get_null_map_data(), NullMap({0, 1, 1})); + EXPECT_EQ(inline_column.get_nested_column().get_data_at(0).to_string(), payload); +} + TEST(JniDataBridgeTest, VariantTypeUsesVariantJniName) { auto variant_type = std::make_shared(); auto nullable_variant_type = make_nullable(variant_type); diff --git a/be/test/format_v2/jni/paimon_jni_reader_test.cpp b/be/test/format_v2/jni/paimon_jni_reader_test.cpp index 8ce9e025a13f9a..af07c7097eda79 100644 --- a/be/test/format_v2/jni/paimon_jni_reader_test.cpp +++ b/be/test/format_v2/jni/paimon_jni_reader_test.cpp @@ -109,6 +109,33 @@ TEST(PaimonJniReaderTest, UsesScanLevelPredicateBeforeLegacySplitPredicate) { EXPECT_EQ(params["paimon_predicate"], "scan-predicate"); } +TEST(PaimonJniReaderTest, UsesDefaultBlobMaxSizeWithoutRuntimeState) { + auto range = make_paimon_jni_range(); + auto scan_params = make_scan_params(); + scan_params.__set_paimon_predicate("serialized-predicate"); + PaimonJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params.at("paimon_blob_max_size"), "10485760"); +} + +TEST(PaimonJniReaderTest, ForwardsBlobMaxSizeFromRuntimeState) { + TQueryOptions query_options; + query_options.__set_paimon_blob_max_size(8); + RuntimeState state {query_options, TQueryGlobals()}; + auto range = make_paimon_jni_range(); + auto scan_params = make_scan_params(); + scan_params.__set_paimon_predicate("serialized-predicate"); + PaimonJniReader reader; + ASSERT_TRUE(init_reader(&reader, &scan_params, &state).ok()); + + std::map params; + ASSERT_TRUE(build_params(&reader, range, ¶ms).ok()); + EXPECT_EQ(params.at("paimon_blob_max_size"), "8"); +} + TEST(PaimonJniReaderTest, ForwardsSerializedTableCacheKey) { auto range = make_paimon_jni_range(); range.table_format_params.paimon_params.__set_paimon_predicate("serialized-predicate"); diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java index 00411e8531285b..0f773db193c625 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java @@ -66,7 +66,8 @@ public enum Type { VARIANT(-1), ARRAY(-1), MAP(-1), - STRUCT(-1); + STRUCT(-1), + FILE(-1); public final int size; @@ -87,27 +88,15 @@ public enum Type { private int scale; public ColumnType(String name, Type type) { - this.name = name; - this.type = type; - this.length = -1; - this.precision = -1; - this.scale = -1; + this(name, type, -1, -1, -1); } public ColumnType(String name, Type type, int length) { - this.name = name; - this.type = type; - this.length = length; - this.precision = -1; - this.scale = -1; + this(name, type, length, -1, -1); } public ColumnType(String name, Type type, int precision, int scale) { - this.name = name; - this.type = type; - this.length = -1; - this.precision = precision; - this.scale = scale; + this(name, type, -1, precision, scale); } public ColumnType(String name, Type type, int length, int precision, int scale) { @@ -116,6 +105,17 @@ public ColumnType(String name, Type type, int length, int precision, int scale) this.length = length; this.precision = precision; this.scale = scale; + if (type == Type.FILE) { + // FILE has a fixed schema, even when nested inside ARRAY, MAP or STRUCT. + childNames = Arrays.asList("uri", "offset", "size", "content_type", "checksum", "inline"); + childTypes = Arrays.asList( + new ColumnType("uri", Type.STRING), + new ColumnType("offset", Type.BIGINT), + new ColumnType("size", Type.BIGINT), + new ColumnType("content_type", Type.STRING), + new ColumnType("checksum", Type.STRING), + new ColumnType("inline", Type.VARBINARY)); + } } public List getChildNames() { @@ -163,7 +163,7 @@ public boolean isVariantType() { } public boolean isComplexType() { - return type == Type.ARRAY || type == Type.MAP || type == Type.STRUCT; + return type == Type.ARRAY || type == Type.MAP || type == Type.STRUCT || type == Type.FILE; } public boolean isArray() { @@ -186,6 +186,10 @@ public boolean isMap() { return type == Type.MAP; } + public boolean isFile() { + return type == Type.FILE; + } + public boolean isStruct() { return type == Type.STRUCT; } @@ -240,10 +244,11 @@ public int metaSize() { case ARRAY: case MAP: case STRUCT: + case FILE: // array & map : [const | nullMap | offsets | ... ] - // struct : [const | nullMap | ... ] + // struct & file : [const | nullMap | ... ] int size = 3; - if (type == Type.STRUCT) { + if (type == Type.STRUCT || type == Type.FILE) { size = 2; } for (ColumnType c : childTypes) { @@ -353,6 +358,9 @@ private static ColumnType parseType(String columnName, String hiveType, boolean case "varbinary": type = Type.VARBINARY; break; + case "file": + type = Type.FILE; + break; case "variant": type = Type.VARIANT; break; diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java index 36c0be38636a9e..1cb3ae20835f59 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java @@ -85,6 +85,10 @@ default byte[] getVariantValue() { throw new UnsupportedOperationException("Variant value is not available"); } + default FileValue getFile() { + throw new UnsupportedOperationException("FILE is not available"); + } + void unpackArray(List values); void unpackMap(List keys, List values); diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/FileValue.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/FileValue.java new file mode 100644 index 00000000000000..1749d085c2a8ce --- /dev/null +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/FileValue.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.jni.vec; + +/** + * A Doris FILE value. Every field is nullable; a null FileValue represents a SQL NULL FILE. + * + *

The inline bytes are not copied. A connector must keep them valid until appendFile returns. + */ +public final class FileValue { + // Canonical child order shared with C++ DataTypeFile and ColumnFile. + public static final int URI_FIELD_INDEX = 0; + public static final int OFFSET_FIELD_INDEX = 1; + public static final int SIZE_FIELD_INDEX = 2; + public static final int CONTENT_TYPE_FIELD_INDEX = 3; + public static final int CHECKSUM_FIELD_INDEX = 4; + public static final int INLINE_FIELD_INDEX = 5; + public static final int FIELD_COUNT = 6; + + private final String uri; + private final Long offset; + private final Long size; + private final String contentType; + private final String checksum; + private final byte[] inline; + + public FileValue(String uri, Long offset, Long size, String contentType, String checksum, byte[] inline) { + this.uri = uri; + this.offset = offset; + this.size = size; + this.contentType = contentType; + this.checksum = checksum; + this.inline = inline; + } + + public String getUri() { + return uri; + } + + public Long getOffset() { + return offset; + } + + public Long getSize() { + return size; + } + + public String getContentType() { + return contentType; + } + + public String getChecksum() { + return checksum; + } + + public byte[] getInline() { + return inline; + } +} diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index 010bf7eba8ff67..1c1bbc093b98e7 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -51,6 +51,8 @@ public class VectorColumn { //add a new flag for const column private boolean isConst = false; + // BE VARBINARY buffers use StringView; Java-written buffers use [length, pointer]. + private boolean varbinaryUsesStringView = false; // NullMap column address private long nullMap; @@ -73,7 +75,7 @@ public class VectorColumn { private int appendIndex; - // For nested column type: String / Array/ Map / Struct + // For nested column type: String / Array / Map / Struct / File private VectorColumn[] childColumns; private VectorColumnVariant variantColumn; @@ -149,6 +151,7 @@ private VectorColumn(long address, int capacity, ColumnType columnType) { // Create readable column, and maybe pass by const column in Readable column private VectorColumn(ColumnType columnType, int numRows, long columnMetaAddress) { + this.varbinaryUsesStringView = true; if (columnType.isUnsupported()) { throw new RuntimeException("Unsupported type for column: " + columnType.getName()); } @@ -183,8 +186,8 @@ private VectorColumn(ColumnType columnType, int numRows, long columnMetaAddress) if (columnType.isComplexType()) { isComplexType = true; - int childRows = numRows; - if (!columnType.isStruct()) { + int childRows = columnType.isFile() ? realRows : numRows; + if (!columnType.isStruct() && !columnType.isFile()) { this.offsets = OffHeap.getLong(null, address); address += 8; childRows = getArrayEndOffset(realRows - 1); @@ -363,7 +366,7 @@ private void reserveCapacity(int newCapacity) { this.data = OffHeap.reallocateMemory(data, oldCapacity * 16L, newCapacity * 16L); } else if (columnType.isVariantType()) { variantColumn.reserveRows(newCapacity); - } else if (!columnType.isStruct()) { + } else if (!columnType.isStruct() && !columnType.isFile()) { throw new RuntimeException("Unhandled type: " + columnType.getName()); } if (!"#stringBytes".equals(columnType.getName())) { @@ -469,6 +472,11 @@ public int appendNull(ColumnType.Type typeValue) { return appendArray(Collections.emptyList()); case MAP: return appendMap(Collections.emptyList(), Collections.emptyList()); + case FILE: + for (VectorColumn child : childColumns) { + child.appendValue(null); + } + return appendIndex++; case STRUCT: return appendStruct(structFieldIndex, null); case BINARY: @@ -1535,6 +1543,100 @@ public HashMap[] getStructColumn(int start, int end) { return result; } + /** Append FILE's canonical six children. Payload bytes are copied into off-heap storage. */ + public int appendFile(FileValue file) { + if (file == null) { + return appendNull(Type.FILE); + } + reserve(appendIndex + 1); + VectorColumn uriColumn = childColumns[FileValue.URI_FIELD_INDEX]; + if (file.getUri() == null) { + uriColumn.appendNull(Type.STRING); + } else { + uriColumn.appendStringAndOffset(file.getUri()); + } + + VectorColumn offsetColumn = childColumns[FileValue.OFFSET_FIELD_INDEX]; + if (file.getOffset() == null) { + offsetColumn.appendNull(Type.BIGINT); + } else { + offsetColumn.appendLong(file.getOffset()); + } + + VectorColumn sizeColumn = childColumns[FileValue.SIZE_FIELD_INDEX]; + if (file.getSize() == null) { + sizeColumn.appendNull(Type.BIGINT); + } else { + sizeColumn.appendLong(file.getSize()); + } + + VectorColumn contentTypeColumn = childColumns[FileValue.CONTENT_TYPE_FIELD_INDEX]; + if (file.getContentType() == null) { + contentTypeColumn.appendNull(Type.STRING); + } else { + contentTypeColumn.appendStringAndOffset(file.getContentType()); + } + + VectorColumn checksumColumn = childColumns[FileValue.CHECKSUM_FIELD_INDEX]; + if (file.getChecksum() == null) { + checksumColumn.appendNull(Type.STRING); + } else { + checksumColumn.appendStringAndOffset(file.getChecksum()); + } + + VectorColumn inlineColumn = childColumns[FileValue.INLINE_FIELD_INDEX]; + if (file.getInline() == null) { + inlineColumn.appendNull(Type.VARBINARY); + } else { + inlineColumn.appendVarbinary(file.getInline()); + } + return appendIndex++; + } + + public FileValue getFile(int rowId) { + rowId = isConst() ? 0 : rowId; + if (isNullAt(rowId)) { + return null; + } + + VectorColumn uriColumn = childColumns[FileValue.URI_FIELD_INDEX]; + String uri = null; + if (!uriColumn.isNullAt(rowId)) { + uri = uriColumn.getStringWithOffset(rowId); + } + + VectorColumn offsetColumn = childColumns[FileValue.OFFSET_FIELD_INDEX]; + Long offset = null; + if (!offsetColumn.isNullAt(rowId)) { + offset = offsetColumn.getLong(rowId); + } + + VectorColumn sizeColumn = childColumns[FileValue.SIZE_FIELD_INDEX]; + Long size = null; + if (!sizeColumn.isNullAt(rowId)) { + size = sizeColumn.getLong(rowId); + } + + VectorColumn contentTypeColumn = childColumns[FileValue.CONTENT_TYPE_FIELD_INDEX]; + String contentType = null; + if (!contentTypeColumn.isNullAt(rowId)) { + contentType = contentTypeColumn.getStringWithOffset(rowId); + } + + VectorColumn checksumColumn = childColumns[FileValue.CHECKSUM_FIELD_INDEX]; + String checksum = null; + if (!checksumColumn.isNullAt(rowId)) { + checksum = checksumColumn.getStringWithOffset(rowId); + } + + VectorColumn inlineColumn = childColumns[FileValue.INLINE_FIELD_INDEX]; + byte[] inline = null; + if (!inlineColumn.isNullAt(rowId)) { + inline = inlineColumn.getBytesVarbinary(rowId); + } + return new FileValue(uri, offset, size, contentType, checksum, inline); + } + public int appendVarbinary(byte[] src) { long addr = OffHeap.allocateMemory(src.length); OffHeap.copyMemory(src, OffHeap.BYTE_ARRAY_OFFSET, null, addr, src.length); @@ -1607,6 +1709,20 @@ public byte[][] getVarBinaryColumn(int start, int end) { } public byte[] getBytesVarbinary(int rowId) { + return varbinaryUsesStringView + ? getBytesVarbinaryFromStringView(rowId) + : getBytesVarbinaryFromLengthAndPointer(rowId); + } + + // Java -> BE: each entry stores an int64 length followed by the payload address. + private byte[] getBytesVarbinaryFromLengthAndPointer(int rowId) { + long entry = data + 16L * rowId; + int size = Math.toIntExact(OffHeap.getLong(null, entry)); + return size == 0 ? new byte[0] : OffHeap.getByte(null, OffHeap.getLong(null, entry + 8), size); + } + + // BE -> Java: decode C++ StringView, including its short-value inline storage. + private byte[] getBytesVarbinaryFromStringView(int rowId) { // Each row is a 16-byte StringView struct at data + 16*rowId // layout: // [0..3] uint32 size @@ -1719,6 +1835,8 @@ public Object[] newObjectContainerArray(ColumnType.Type type, int size) { case VARCHAR: case STRING: return new String[size]; + case FILE: + return new FileValue[size]; case ARRAY: return new ArrayList[size]; case MAP: @@ -1794,6 +1912,14 @@ public void appendObjectColumn(Object[] batch, boolean isNullable) { case MAP: appendMap((Map[]) batch, isNullable); break; + case FILE: + if (!isNullable) { + checkNullable(batch, batch.length); + } + for (Object value : batch) { + appendFile((FileValue) value); + } + break; case STRUCT: appendStruct((Map[]) batch, isNullable); break; @@ -1853,6 +1979,12 @@ public Object[] getObjectColumn(int start, int end) { return getArrayColumn(start, end); case MAP: return getMapColumn(start, end); + case FILE: + FileValue[] files = new FileValue[end - start]; + for (int i = start; i < end; ++i) { + files[i - start] = getFile(i); + } + return files; case STRUCT: return getStructColumn(start, end); case BINARY: @@ -1947,6 +2079,9 @@ public void appendValue(ColumnValue o) { appendMap(keys, values); break; } + case FILE: + appendFile(o.getFile()); + break; case STRUCT: { List values = new ArrayList<>(); o.unpackStruct(structFieldIndex, values); @@ -2074,6 +2209,7 @@ public void dump(StringBuilder sb, int i) { sb.append("}"); break; } + case FILE: case STRUCT: { sb.append("{"); for (int fieldIndex = 0; fieldIndex < childColumns.length; ++fieldIndex) { diff --git a/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnFileTest.java b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnFileTest.java new file mode 100644 index 00000000000000..23d72c4aaa15b1 --- /dev/null +++ b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnFileTest.java @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.jni.vec; + +import org.apache.doris.common.jni.utils.OffHeap; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class VectorColumnFileTest { + @BeforeClass + public static void setUpClass() { + OffHeap.setTesting(); + } + + @Test + public void testFileTypeInsideNestedTypes() { + ColumnType file = ColumnType.parseType("image", "file"); + Assert.assertTrue(file.isFile()); + Assert.assertFalse(file.isStruct()); + Assert.assertTrue(file.isComplexType()); + Assert.assertEquals(Arrays.asList("uri", "offset", "size", "content_type", "checksum", "inline"), + file.getChildNames()); + Assert.assertEquals(FileValue.FIELD_COUNT, file.getChildTypes().size()); + Assert.assertEquals(23, file.metaSize()); + Assert.assertTrue(ColumnType.parseType("a", "array").getChildTypes().get(0).isFile()); + Assert.assertTrue(ColumnType.parseType("m", "map").getChildTypes().get(1).isFile()); + Assert.assertTrue(ColumnType.parseType("s", "struct").getChildTypes().get(0).isFile()); + Assert.assertTrue(ColumnType.parseTypeWithEncodedStructFields("s", "struct<$aW1hZ2U=:file>") + .getChildTypes().get(0).isFile()); + } + + @Test + public void testAppendInlineDescriptorNullAndEmptyFile() { + ColumnType fileType = ColumnType.parseType("image", "file"); + VectorTable table = VectorTable.createWritableTable( + new ColumnType[] {fileType}, new String[] {"image"}, 1); + try { + VectorColumn column = table.getColumn(0); + byte[] payload = "hello world".getBytes(StandardCharsets.UTF_8); + column.appendFile(new FileValue(null, null, null, null, null, payload)); + column.appendFile(new FileValue("s3://bucket/data.blob", 17L, 11L, "image/png", "crc", null)); + column.appendValue(null); + column.appendFile(new FileValue(null, null, null, null, null, new byte[0])); + + Assert.assertArrayEquals(payload, column.getFile(0).getInline()); + Assert.assertEquals("image/png", column.getFile(1).getContentType()); + Assert.assertEquals("crc", column.getFile(1).getChecksum()); + Assert.assertNull(column.getFile(2)); + Assert.assertArrayEquals(new byte[0], column.getFile(3).getInline()); + long meta = table.getMetaAddress(); + Assert.assertEquals(4, OffHeap.getLong(null, meta)); + Assert.assertArrayEquals(new boolean[] {false, false, true, false}, + OffHeap.getBoolean(null, OffHeap.getLong(null, meta + 8), 4)); + long uriMeta = childMeta(table, FileValue.URI_FIELD_INDEX); + Assert.assertArrayEquals(new boolean[] {true, false, true, true}, + OffHeap.getBoolean(null, OffHeap.getLong(null, uriMeta), 4)); + long offsetData = OffHeap.getLong(null, childMeta(table, FileValue.OFFSET_FIELD_INDEX) + 8); + long sizeData = OffHeap.getLong(null, childMeta(table, FileValue.SIZE_FIELD_INDEX) + 8); + Assert.assertEquals(17L, OffHeap.getLong(null, offsetData + 8)); + Assert.assertEquals(11L, OffHeap.getLong(null, sizeData + 8)); + long inlineMeta = childMeta(table, FileValue.INLINE_FIELD_INDEX); + Assert.assertArrayEquals(new boolean[] {false, true, true, false}, + OffHeap.getBoolean(null, OffHeap.getLong(null, inlineMeta), 4)); + // Java -> BE VARBINARY uses [length, pointer], including short payloads. + long inlineData = OffHeap.getLong(null, inlineMeta + 8); + Assert.assertEquals(payload.length, OffHeap.getLong(null, inlineData)); + Assert.assertArrayEquals(payload, + OffHeap.getByte(null, OffHeap.getLong(null, inlineData + 8), payload.length)); + Assert.assertEquals(0, OffHeap.getLong(null, inlineData + 3 * 16)); + + table.reset(); + column.appendFile(new FileValue("file:/new.blob", 0L, null, null, null, null)); + Assert.assertEquals(1, OffHeap.getLong(null, table.getMetaAddress())); + Assert.assertFalse(column.isNullAt(0)); + Assert.assertTrue(OffHeap.getBoolean(null, + OffHeap.getLong(null, childMeta(table, FileValue.INLINE_FIELD_INDEX)))); + } finally { + table.close(); + } + } + + @Test + public void testObjectBatchPreservesNullableMetadata() { + VectorTable table = VectorTable.createWritableTable( + new ColumnType[] {ColumnType.parseType("image", "file")}, new String[] {"image"}, 1); + try { + FileValue descriptor = new FileValue("file:/a.blob", 0L, null, null, null, null); + table.appendData(0, new FileValue[] {descriptor, null}, true); + FileValue[] values = (FileValue[]) table.getColumn(0).getObjectColumn(0, 2); + Assert.assertEquals(descriptor.getUri(), values[0].getUri()); + Assert.assertEquals(Long.valueOf(0), values[0].getOffset()); + Assert.assertNull(values[0].getSize()); + Assert.assertNull(values[0].getInline()); + Assert.assertNull(values[1]); + Assert.assertThrows(RuntimeException.class, + () -> table.appendData(0, new FileValue[] {null}, false)); + } finally { + table.close(); + } + } + + @Test + public void testFileInsideArrayMapAndStruct() { + ColumnType[] types = new ColumnType[] { + ColumnType.parseType("a", "array"), + ColumnType.parseType("m", "map"), + ColumnType.parseType("s", "struct")}; + VectorTable table = VectorTable.createWritableTable(types, new String[] {"a", "m", "s"}, 1); + try { + byte[] bytes = new byte[] {0, 42, (byte) 255}; + FileValue file = new FileValue(null, null, null, null, null, bytes); + table.appendData(0, new List[] {Arrays.asList(file, null)}, true); + table.appendData(1, new Map[] {Collections.singletonMap("image", file)}, true); + table.appendData(2, new Map[] {Collections.singletonMap("image", file)}, true); + List array = (List) table.getColumn(0).getObjectColumn(0, 1)[0]; + Assert.assertArrayEquals(bytes, ((FileValue) array.get(0)).getInline()); + Assert.assertNull(array.get(1)); + Map map = (Map) table.getColumn(1).getObjectColumn(0, 1)[0]; + Map struct = (Map) table.getColumn(2).getObjectColumn(0, 1)[0]; + Assert.assertArrayEquals(bytes, ((FileValue) map.get("image")).getInline()); + Assert.assertArrayEquals(bytes, ((FileValue) struct.get("image")).getInline()); + } finally { + table.close(); + } + } + + @Test + public void testReadFileFromBeStringViewLayout() { + ColumnType fileType = ColumnType.parseType("image", "file"); + // BE -> Java includes a const flag per column. Model a FILE with all-null + // metadata and a short VARBINARY payload stored inline in a C++ StringView. + long meta = OffHeap.allocateMemory(fileType.metaSize() * 8L); + long nullMaps = OffHeap.allocateMemory(7); + long payload = OffHeap.allocateMemory(16); + long emptyData = OffHeap.allocateMemory(8); + try { + OffHeap.setMemory(meta, (byte) 0, fileType.metaSize() * 8L); + OffHeap.setMemory(nullMaps, (byte) 1, 7); + OffHeap.putByte(null, nullMaps, (byte) 0); + OffHeap.putByte(null, nullMaps + 6, (byte) 0); + OffHeap.setMemory(emptyData, (byte) 0, 8); + OffHeap.setMemory(payload, (byte) 0, 16); + byte[] bytes = "hello".getBytes(StandardCharsets.UTF_8); + OffHeap.putInt(null, payload, bytes.length); + OffHeap.copyMemory(bytes, OffHeap.BYTE_ARRAY_OFFSET, null, payload + 4, bytes.length); + OffHeap.putLong(null, meta + 8, nullMaps); + long child = meta + 16; + for (int i = 0; i < FileValue.FIELD_COUNT; i++) { + OffHeap.putLong(null, child + 8, nullMaps + i + 1); + OffHeap.putLong(null, child + 16, emptyData); + if (fileType.getChildTypes().get(i).isStringType()) { + OffHeap.putLong(null, child + 24, emptyData); + } + if (i == FileValue.INLINE_FIELD_INDEX) { + OffHeap.putLong(null, child + 16, payload); + } + child += fileType.getChildTypes().get(i).metaSize() * 8L; + } + VectorColumn column = VectorColumn.createReadableColumn(fileType, 1, meta); + FileValue file = column.getFile(0); + Assert.assertNull(file.getUri()); + Assert.assertNull(file.getSize()); + Assert.assertArrayEquals(bytes, file.getInline()); + OffHeap.putLong(null, meta, 1L); // constant FILE owns one physical row + VectorColumn constant = VectorColumn.createReadableColumn(fileType, 5, meta); + Assert.assertArrayEquals(bytes, constant.getFile(4).getInline()); + } finally { + // Readable columns borrow BE buffers; the owner releases them. + OffHeap.freeMemory(emptyData); + OffHeap.freeMemory(payload); + OffHeap.freeMemory(nullMaps); + OffHeap.freeMemory(meta); + } + } + + private long childMeta(VectorTable table, int childIndex) { + long address = table.getMetaAddress() + 16; // row count and FILE null map + for (int i = 0; i < childIndex; i++) { + // Java -> BE omits the const flags used by BE -> Java. + address += (table.getColumn(0).getColumnType().getChildTypes().get(i).metaSize() - 1) * 8L; + } + return address; + } +} diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index 3c678e7f7df6d0..aa60858e6d1d79 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -19,22 +19,19 @@ import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.ColumnValue; +import org.apache.doris.common.jni.vec.FileValue; -import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Blob; import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.data.BlobRef; import org.apache.paimon.data.DataGetters; -import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.variant.Variant; import org.apache.paimon.types.ArrayType; -import org.apache.paimon.types.BlobType; import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; @@ -56,15 +53,6 @@ public class PaimonColumnValue implements ColumnValue { private static final Logger LOG = LoggerFactory.getLogger(PaimonColumnValue.class); static final long DEFAULT_BLOB_MAX_SIZE = 10L * 1024 * 1024; private static final Map DORIS_TIME_ZONE_ALIASES; - // JNI transports Doris FILE as a six-child STRUCT. This is only the adapter's - // child schema; the Paimon reader must continue to read the logical BLOB type. - private static final RowType FILE_ROW_TYPE = DataTypes.ROW( - DataTypes.FIELD(0, "uri", DataTypes.STRING()), - DataTypes.FIELD(1, "offset", DataTypes.BIGINT()), - DataTypes.FIELD(2, "size", DataTypes.BIGINT()), - DataTypes.FIELD(3, "content_type", DataTypes.STRING()), - DataTypes.FIELD(4, "checksum", DataTypes.STRING()), - DataTypes.FIELD(5, "inline", DataTypes.BYTES())); static { Map aliases = new HashMap<>(ZoneId.SHORT_IDS); @@ -88,7 +76,6 @@ public class PaimonColumnValue implements ColumnValue { private Variant materializedVariant; private boolean blobAsDescriptor; private long blobMaxSize = DEFAULT_BLOB_MAX_SIZE; - private GenericRow materializedBlob; // Keep these caches lazy so scalar columns do not pay for complex-type reuse bookkeeping. private List arrayValues; private List mapKeys; @@ -120,7 +107,6 @@ public void setIdx( ColumnType dorisType, DataType dataType, PaimonVariantProjection variantProjection) { - clearMaterializedBlob(); this.idx = idx; this.dorisType = dorisType; this.dataType = dataType; @@ -129,7 +115,6 @@ public void setIdx( } public void setOffsetRow(InternalRow record) { - clearMaterializedBlob(); this.record = record; this.materializedVariant = null; } @@ -309,16 +294,9 @@ public void unpackMap(List keys, List values) { @Override public void unpackStruct(List structFieldIndex, List values) { - RowType rowType; - InternalRow row; - if (dataType instanceof BlobType) { - rowType = FILE_ROW_TYPE; - row = getBlobFileRow(); - } else { - rowType = (RowType) dataType; - // Projection entries are original child indexes, so keep the full RowType arity. - row = record.getRow(idx, rowType.getFieldCount()); - } + RowType rowType = (RowType) dataType; + // Projection entries are original child indexes, so keep the full RowType arity. + InternalRow row = record.getRow(idx, rowType.getFieldCount()); if (structValues == null) { structValues = new ArrayList<>(); } @@ -328,22 +306,16 @@ public void unpackStruct(List structFieldIndex, List value } } - private InternalRow getBlobFileRow() { - if (materializedBlob == null) { - materializedBlob = new GenericRow(FILE_ROW_TYPE.getFieldCount()); - } + @Override + public FileValue getFile() { Blob blob = record.getBlob(idx); if (blobAsDescriptor) { - // Use the SDK's typed descriptor API. Payload bytes are never inspected to guess - // whether they contain a descriptor, and this path does not read the payload. + // Use the SDK's typed descriptor API without reading the payload. BlobDescriptor descriptor = blob.toDescriptor(); - materializedBlob.setField(0, BinaryString.fromString(descriptor.uri())); - materializedBlob.setField(1, descriptor.offset()); - materializedBlob.setField(2, descriptor.length() < 0 ? null : descriptor.length()); - } else { - materializedBlob.setField(5, readBlobPayload(blob)); + return new FileValue(descriptor.uri(), descriptor.offset(), + descriptor.length() < 0 ? null : descriptor.length(), null, null, null); } - return materializedBlob; + return new FileValue(null, null, null, null, null, readBlobPayload(blob)); } private byte[] readBlobPayload(Blob blob) { @@ -368,16 +340,6 @@ private void checkBlobSize(long size) { } } - private void clearMaterializedBlob() { - if (materializedBlob != null) { - // VectorColumn consumes children synchronously. Drop the prior payload as soon as - // this wrapper advances, rather than retaining a large byte array across columns. - for (int i = 0; i < FILE_ROW_TYPE.getFieldCount(); i++) { - materializedBlob.setField(i, null); - } - } - } - private PaimonColumnValue reuseColumnValue( List cache, int cacheIndex, DataGetters childRecord, int childIndex, ColumnType childDorisType, DataType childPaimonType) { @@ -400,7 +362,6 @@ private PaimonColumnValue reuseColumnValue( private void reset( DataGetters record, int idx, ColumnType dorisType, DataType dataType, ZoneId timeZone) { - clearMaterializedBlob(); this.record = record; this.idx = idx; this.dorisType = dorisType; @@ -426,6 +387,5 @@ private void clearChildCaches() { mapKeys = null; mapValues = null; structValues = null; - materializedBlob = null; } } diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java index 88393c50c6a35a..677cc990adf16b 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java @@ -41,8 +41,8 @@ private PaimonReadTypeProjection() { } static DataType project(DataType tableType, ColumnType requiredType) { - // FILE uses STRUCT on the JNI wire, but its six metadata children are constructed - // after reading BLOB. Asking Paimon to project those children would change its schema. + // FILE children are constructed by the JNI adapter after reading BLOB. + // Keep the logical Paimon BLOB type intact for the SDK reader. if (tableType instanceof BlobType) { return tableType; } diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java index be457571f0be87..47ae36acd40d7b 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java @@ -23,6 +23,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobType; import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; @@ -66,7 +67,9 @@ public static ColumnType fromPaimonType(String columnName, DataType type) { ColumnType columnType = new ColumnType(columnName, paimonColumnType.getType(), paimonColumnType.getLength(), paimonColumnType.getPrecision(), paimonColumnType.getScale()); - columnType.setChildTypes(paimonColumnType.getChildTypes()); + if (!columnType.isFile()) { + columnType.setChildTypes(paimonColumnType.getChildTypes()); + } return columnType; } @@ -169,6 +172,11 @@ public PaimonColumnType visit(LocalZonedTimestampType localZonedTimestampType) { return paimonColumnType; } + @Override + public PaimonColumnType visit(BlobType blobType) { + return new PaimonColumnType(Type.FILE); + } + @Override public PaimonColumnType visit(VariantType variantType) { return new PaimonColumnType(Type.VARIANT); diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java index c0651163c7d5a7..75cc5fa326b653 100644 --- a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java @@ -17,10 +17,15 @@ package org.apache.doris.paimon; +import org.apache.doris.common.jni.utils.OffHeap; import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.ColumnValue; +import org.apache.doris.common.jni.vec.FileValue; +import org.apache.doris.common.jni.vec.VectorTable; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Blob; +import org.apache.paimon.data.BlobDescriptor; import org.apache.paimon.data.GenericArray; import org.apache.paimon.data.GenericMap; import org.apache.paimon.data.GenericRow; @@ -30,6 +35,7 @@ import org.apache.paimon.data.variant.GenericVariant; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.BlobType; import org.apache.paimon.types.DataType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; @@ -42,6 +48,7 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; @@ -58,6 +65,96 @@ public class PaimonColumnValueTest { private final ColumnType dorisStructType = ColumnType.parseType("s", "struct"); + @Test + public void testInlineFileContainsOnlyPayload() { + byte[] payload = "hello world".getBytes(StandardCharsets.UTF_8); + PaimonColumnValue value = createBlobValue(Blob.fromData(payload)); + FileValue file = value.getFile(); + Assert.assertArrayEquals(payload, file.getInline()); + Assert.assertNull(file.getUri()); + Assert.assertNull(file.getOffset()); + Assert.assertNull(file.getSize()); + Assert.assertNull(file.getContentType()); + Assert.assertNull(file.getChecksum()); + Assert.assertTrue(PaimonTypeUtils.fromPaimonType("image", new BlobType()).isFile()); + } + + @Test + public void testDescriptorDoesNotReadPayloadOrApplyInlineLimit() { + for (long length : new long[] {11, -1}) { + PaimonColumnValue value = createBlobValue(descriptorBlob(length)); + value.setBlobAsDescriptor(true); + value.setBlobMaxSize(1); + FileValue file = value.getFile(); + Assert.assertEquals("file:/a.blob", file.getUri()); + Assert.assertEquals(Long.valueOf(17), file.getOffset()); + Assert.assertEquals(length < 0 ? null : Long.valueOf(length), file.getSize()); + Assert.assertNull(file.getInline()); + } + } + + @Test + public void testInlineSizeLimitAndEmptyPayload() { + PaimonColumnValue value = createBlobValue(Blob.fromData(new byte[] {1, 2})); + value.setBlobMaxSize(2); + Assert.assertArrayEquals(new byte[] {1, 2}, value.getFile().getInline()); + value.setBlobMaxSize(1); + Assert.assertThrows(IllegalArgumentException.class, value::getFile); + value.setOffsetRow(GenericRow.of(Blob.fromData(new byte[0]))); + Assert.assertArrayEquals(new byte[0], value.getFile().getInline()); + value.setOffsetRow(GenericRow.of((Object) null)); + Assert.assertTrue(value.isNull()); + } + + @Test + public void testKnownLengthLimitCheckedBeforeReading() { + PaimonColumnValue value = createBlobValue(descriptorBlob(11)); + value.setBlobMaxSize(10); + Assert.assertThrows(IllegalArgumentException.class, value::getFile); + } + + @Test + public void testNestedFileUsesFileGetter() { + RowType rowType = RowType.of(new DataType[] {new BlobType()}, new String[] {"image"}); + PaimonColumnValue value = new PaimonColumnValue( + GenericRow.of(GenericRow.of(Blob.fromData(new byte[] {42}))), 0, + ColumnType.parseType("s", "struct"), rowType, "UTC"); + List children = new ArrayList<>(); + value.unpackStruct(Collections.singletonList(0), children); + Assert.assertArrayEquals(new byte[] {42}, children.get(0).getFile().getInline()); + } + + @Test + public void testAppendBlobThroughFileColumnValue() { + OffHeap.setTesting(); + VectorTable table = VectorTable.createWritableTable( + new ColumnType[] {ColumnType.parseType("image", "file")}, new String[] {"image"}, 1); + try { + byte[] payload = new byte[] {0, 42, (byte) 255}; + PaimonColumnValue value = createBlobValue(Blob.fromData(payload)); + table.appendData(0, value); + value.setOffsetRow(GenericRow.of(descriptorBlob(11))); + value.setBlobAsDescriptor(true); + table.appendData(0, value); + Assert.assertArrayEquals(payload, table.getColumn(0).getFile(0).getInline()); + Assert.assertEquals("file:/a.blob", table.getColumn(0).getFile(1).getUri()); + Assert.assertNull(table.getColumn(0).getFile(1).getInline()); + } finally { + table.close(); + } + } + + private PaimonColumnValue createBlobValue(Blob blob) { + return new PaimonColumnValue(GenericRow.of(blob), 0, + ColumnType.parseType("image", "file"), new BlobType(), "UTC"); + } + + private Blob descriptorBlob(long length) { + return Blob.fromDescriptor(uri -> { + throw new AssertionError("Payload must not be read"); + }, new BlobDescriptor("file:/a.blob", 17, length)); + } + @Test public void testUnpackNonLeadingStructField() { PaimonColumnValue structValue = createStructValue(); diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonReadTypeProjectionTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonReadTypeProjectionTest.java index bc4acf2e1bcbf9..808e8838c426f1 100644 --- a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonReadTypeProjectionTest.java +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonReadTypeProjectionTest.java @@ -20,7 +20,9 @@ import org.apache.doris.common.jni.vec.ColumnType; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BlobType; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; @@ -30,6 +32,17 @@ import java.util.Arrays; public class PaimonReadTypeProjectionTest { + @Test + public void testFileProjectionPreservesPaimonBlobType() { + BlobType blob = new BlobType(); + Assert.assertSame(blob, + PaimonReadTypeProjection.project(blob, ColumnType.parseType("image", "file"))); + RowType row = RowType.of(new DataType[] {blob}, new String[] {"image"}); + RowType projected = (RowType) PaimonReadTypeProjection.project( + row, ColumnType.parseType("s", "struct")); + Assert.assertEquals(blob, projected.getTypeAt(0)); + } + @Test public void testProjectNestedRowArrayAndMap() { RowType profileType = new RowType(false, Arrays.asList( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java index aa1d6415adedd8..00fb5f43a64668 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java @@ -48,6 +48,7 @@ public class PaimonReaderOptionsTest { @Test void testSupportsSafeBatchReadOptions() { Assertions.assertEquals(ImmutableSet.of( + CoreOptions.BLOB_AS_DESCRIPTOR.key(), CoreOptions.READ_BATCH_SIZE.key(), CoreOptions.FILE_READER_ASYNC_THRESHOLD.key(), CoreOptions.FILE_INDEX_READ_ENABLED.key(), @@ -65,9 +66,21 @@ void testSupportsSafeBatchReadOptions() { AbstractPaimonProperties.TABLE_OPTION_PREFIX + "scan.plan-sort-partition", "true"))); } + @Test + void testSupportsBlobAsDescriptorOption() { + for (String value : new String[] {"true", "false"}) { + Assertions.assertDoesNotThrow(() -> PaimonReaderOptions.validateCatalogProperties( + ImmutableMap.of(AbstractPaimonProperties.TABLE_OPTION_PREFIX + + CoreOptions.BLOB_AS_DESCRIPTOR.key(), value))); + Assertions.assertDoesNotThrow(() -> PaimonReaderOptions.validateEffectiveTableOptions( + ImmutableMap.of(CoreOptions.BLOB_AS_DESCRIPTOR.key(), value))); + } + } + @Test void testRejectsInvalidSafeBatchReadOptions() { for (Map options : new Map[] { + ImmutableMap.of(CoreOptions.BLOB_AS_DESCRIPTOR.key(), "not-a-boolean"), ImmutableMap.of("file-index.read.enabled", "not-a-boolean"), ImmutableMap.of("source.split.target-size", "0 B"), ImmutableMap.of("source.split.open-file-cost", "-1 B"), @@ -81,6 +94,8 @@ void testRejectsInvalidSafeBatchReadOptions() { @Test void testRejectUnsafeOptionsFromCreateOrAlterProperties() { for (Map properties : new Map[] { + ImmutableMap.of(AbstractPaimonProperties.TABLE_OPTION_PREFIX + + CoreOptions.BLOB_AS_DESCRIPTOR.key(), "not-a-boolean"), ImmutableMap.of(AbstractPaimonProperties.TABLE_OPTION_PREFIX + "branch", "archive"), ImmutableMap.of(AbstractPaimonProperties.TABLE_OPTION_PREFIX + "read.batch-size", "0"), ImmutableMap.of(AbstractPaimonProperties.TABLE_OPTION_PREFIX diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 003036bfbde1dc..305a616202be9f 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -514,8 +514,9 @@ struct TQueryOptions { // enable_paimon_cpp_reader. 229: optional bool enable_paimon_rust_reader = false; + // IDs 230-236 are allocated on master; do not reuse them in release branches. // Maximum payload bytes per Paimon BLOB materialized as FILE inline. Descriptors are exempt. - 230: optional i64 paimon_blob_max_size = 10485760; + 237: optional i64 paimon_blob_max_size = 10485760; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. From dd1aba057e3f1f11a099d61577428261fa3eb336 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Fri, 16 Oct 2026 23:31:35 +0800 Subject: [PATCH 5/5] update --- be/src/format_v2/blob/blob_file_reader.cpp | 6 +++ be/src/format_v2/blob/blob_file_reader.h | 4 ++ .../blob/paimon_blob_column_reader.cpp | 13 +++++-- .../blob/paimon_blob_column_reader.h | 5 ++- .../blob/paimon_blob_materializer.cpp | 12 +++++- .../format_v2/blob/paimon_blob_materializer.h | 5 ++- be/src/format_v2/table/paimon_reader.cpp | 4 +- .../blob/paimon_blob_column_reader_test.cpp | 38 +++++++++++++++++++ .../blob/paimon_blob_materializer_test.cpp | 23 +++++++++++ .../format_v2/blob/paimon_blob_test_util.h | 1 + .../doris/common/jni/vec/VectorColumn.java | 5 +-- .../doris/paimon/PaimonColumnValue.java | 29 +++++++++++--- 12 files changed, 129 insertions(+), 16 deletions(-) diff --git a/be/src/format_v2/blob/blob_file_reader.cpp b/be/src/format_v2/blob/blob_file_reader.cpp index 46f141710ccf61..a01e89730e3753 100644 --- a/be/src/format_v2/blob/blob_file_reader.cpp +++ b/be/src/format_v2/blob/blob_file_reader.cpp @@ -90,6 +90,12 @@ constexpr int64_t BLOB_FOOTER_SIZE = 5; // Index length (4) followed by format v } // namespace +BlobFileReader::~BlobFileReader() { + if (_reader) { + _reader->collect_profile_before_close(); + } +} + Status BlobFileReader::_read_exact(int64_t offset, char* data, size_t size, const io::IOContext* io_ctx) const { if (offset < 0 || uint64_t(offset) > _reader->size() || diff --git a/be/src/format_v2/blob/blob_file_reader.h b/be/src/format_v2/blob/blob_file_reader.h index 7697b36aa6b3bb..ef8627f38be76b 100644 --- a/be/src/format_v2/blob/blob_file_reader.h +++ b/be/src/format_v2/blob/blob_file_reader.h @@ -30,6 +30,8 @@ namespace doris::format { // table row ids, field ids, projection, and SQL FILE values belong to the caller. class BlobFileReader { public: + ~BlobFileReader(); + struct ByteRange { // Absolute byte offset of the payload in this .blob file, excluding record framing. int64_t offset; @@ -38,6 +40,8 @@ class BlobFileReader { }; // Read and decode the footer/index only. Data access requires a successful open(). + // Owns the supplied reader and flushes its I/O profile on destruction, + // including when open() fails after performing I/O. Status open(io::FileReaderSPtr reader, int64_t expected_rows, const io::IOContext* io_ctx); // row is zero-based within this .blob file. SQL NULL returns nullopt, while an empty // non-NULL value returns a ByteRange with size 0. Descriptor reads need only this index. diff --git a/be/src/format_v2/blob/paimon_blob_column_reader.cpp b/be/src/format_v2/blob/paimon_blob_column_reader.cpp index 5fdc9a1847f357..4d7f8f749b6408 100644 --- a/be/src/format_v2/blob/paimon_blob_column_reader.cpp +++ b/be/src/format_v2/blob/paimon_blob_column_reader.cpp @@ -25,6 +25,7 @@ #include "core/pod_array.h" #include "format_v2/blob/blob_file_reader.h" #include "io/file_factory.h" +#include "io/fs/tracing_file_reader.h" #include "io/io_common.h" namespace doris::format::paimon { @@ -87,16 +88,22 @@ Status PaimonBlobColumnReader::_open_blob_file(BlobFileState* blob_file, description.file_cache_admission = context.file_cache_admission; auto reader = FileFactory::create_file_reader( context.system_properties, description, - FileFactory::get_reader_options(context.runtime_state, description)); + FileFactory::get_reader_options(context.runtime_state, description), context.profile); if (!reader.has_value()) { return std::move(reader).error(); } if (reader.value()->size() != uint64_t(description.file_size)) { return Status::Corruption("BLOB file size differs from its manifest: {}", description.path); } + auto file_reader = std::move(reader).value(); + // Match ordinary file reads so footer, index and payload I/O all reach scanner accounting. + if (context.io_ctx && context.io_ctx->file_reader_stats) { + file_reader = std::make_shared(std::move(file_reader), + context.io_ctx->file_reader_stats); + } auto blob = std::make_unique(); - RETURN_IF_ERROR(blob->open(std::move(reader).value(), blob_file->blob_metadata.row_count, - context.io_ctx)); + RETURN_IF_ERROR( + blob->open(std::move(file_reader), blob_file->blob_metadata.row_count, context.io_ctx)); blob_file->blob_reader = std::move(blob); return Status::OK(); } diff --git a/be/src/format_v2/blob/paimon_blob_column_reader.h b/be/src/format_v2/blob/paimon_blob_column_reader.h index af2fcdd4ff61d5..c20c9df7053c64 100644 --- a/be/src/format_v2/blob/paimon_blob_column_reader.h +++ b/be/src/format_v2/blob/paimon_blob_column_reader.h @@ -27,6 +27,7 @@ #include "gen_cpp/PlanNodes_types.h" namespace doris { +class RuntimeProfile; class RuntimeState; namespace io { struct FileSystemProperties; @@ -46,11 +47,13 @@ namespace doris::format::paimon { // This reader has no knowledge of output Block positions. class PaimonBlobColumnReader { public: - // Borrowed only for read_blob(); no owning TableReader state is retained. + // The context is borrowed for read_blob(). Scanner-owned profile and I/O statistics + // must outlive this reader, since the opened file readers retain them. struct ReadContext { const io::FileSystemProperties& system_properties; RuntimeState* runtime_state; const io::IOContext* io_ctx; + RuntimeProfile* profile; const std::string& fs_name; bool file_cache_admission; // Per-value inline payload limit; descriptor output does not read payload bytes. diff --git a/be/src/format_v2/blob/paimon_blob_materializer.cpp b/be/src/format_v2/blob/paimon_blob_materializer.cpp index 544c52ebfd454e..56970a180b6b99 100644 --- a/be/src/format_v2/blob/paimon_blob_materializer.cpp +++ b/be/src/format_v2/blob/paimon_blob_materializer.cpp @@ -23,10 +23,18 @@ #include "core/block/block.h" #include "core/data_type/data_type_nullable.h" #include "format_v2/blob/paimon_blob_column_reader.h" +#include "runtime/file_scan_profile.h" namespace doris::format::paimon { -Status PaimonBlobMaterializer::init(const std::map& options) { +Status PaimonBlobMaterializer::init(const std::map& options, + RuntimeProfile* profile) { + _materialize_timer = nullptr; + if (profile) { + _materialize_timer = ADD_CHILD_TIMER_WITH_LEVEL( + profile, "PaimonBlobMaterializeTime", + file_scan_profile::parent_or_root(profile, file_scan_profile::TABLE_READER), 1); + } _blob_as_descriptor = false; const auto it = options.find("blob-as-descriptor"); if (it != options.end()) { @@ -96,6 +104,8 @@ Status PaimonBlobMaterializer::materialize(const ColumnInt64& anchor_row_positio if (!has_blob_columns() || table_block->rows() == 0) { return Status::OK(); } + // Includes lazy file opening, index decoding, payload reads and FILE construction. + SCOPED_TIMER(_materialize_timer); if (anchor_row_positions.size() != table_block->rows()) { return Status::InternalError("BLOB row positions and Parquet output are not aligned"); } diff --git a/be/src/format_v2/blob/paimon_blob_materializer.h b/be/src/format_v2/blob/paimon_blob_materializer.h index a44504ef783d2f..4f814ac2a2aeff 100644 --- a/be/src/format_v2/blob/paimon_blob_materializer.h +++ b/be/src/format_v2/blob/paimon_blob_materializer.h @@ -27,6 +27,7 @@ #include "format_v2/blob/paimon_blob_column_reader.h" #include "format_v2/column_data.h" #include "gen_cpp/PlanNodes_types.h" +#include "runtime/runtime_profile.h" namespace doris { class Block; @@ -41,7 +42,8 @@ class PaimonBlobMaterializer { public: using ReadContext = PaimonBlobColumnReader::ReadContext; - Status init(const std::map& options); + Status init(const std::map& options, + RuntimeProfile* profile = nullptr); // Clears the previous split, validates row ranges and resolves projected FILE columns. // This does not open BLOB files. An absent blob_files field disables materialization; // a present but empty list still creates readers that fill missing nullable columns with NULL. @@ -55,6 +57,7 @@ class PaimonBlobMaterializer { const std::set& blob_column_names() const { return _blob_column_names; } private: + RuntimeProfile::Counter* _materialize_timer = nullptr; bool _blob_as_descriptor = false; // Logical range of the complete Parquet anchor. All byte splits use the same origin; // ROW_POSITION remains relative to the whole file, including rows skipped by filtering. diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index d90560bc6493c6..0945cd88e8b0c5 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -248,7 +248,8 @@ Status PaimonReader::init(format::TableReadOptions&& options) { const std::map empty_options; return _blob_materializer->init(_scan_params && _scan_params->__isset.paimon_options ? _scan_params->paimon_options - : empty_options); + : empty_options, + _scanner_profile); } Status PaimonReader::abort_split() { @@ -345,6 +346,7 @@ Status PaimonReader::materialize_virtual_columns(Block* table_block) { {.system_properties = *_system_properties, .runtime_state = _runtime_state, .io_ctx = _io_ctx.get(), + .profile = _scanner_profile, .fs_name = _current_file_range_desc.fs_name, .file_cache_admission = !_current_file_range_desc.__isset.file_cache_admission || _current_file_range_desc.file_cache_admission, diff --git a/be/test/format_v2/blob/paimon_blob_column_reader_test.cpp b/be/test/format_v2/blob/paimon_blob_column_reader_test.cpp index 78ebe6d99764fa..a48a5e9e246401 100644 --- a/be/test/format_v2/blob/paimon_blob_column_reader_test.cpp +++ b/be/test/format_v2/blob/paimon_blob_column_reader_test.cpp @@ -30,6 +30,44 @@ namespace { class PaimonBlobColumnReaderTest : public blob_test::PaimonBlobTest {}; +TEST_F(PaimonBlobColumnReaderTest, AccountsForFooterIndexAndPayloadAcrossBatchesAndFiles) { + auto first = write_blob("first.blob", 7, 100, {"hello", "", std::nullopt}); + auto second = write_blob("second.blob", 7, 103, {"world"}); + io::FileReaderStats stats; + _io_context.file_reader_stats = &stats; + auto context = read_context(); // Local FileFactory reads, with no file cache. + PaimonBlobColumnReader reader(blob_column("image", 7), {&first, &second}, false); + MutableColumnPtr result; + ASSERT_TRUE(reader.read_blob(*row_positions({0}), 100, context, &result).ok()); + EXPECT_EQ(stats.read_calls, 3); // Footer, index, first complete record. + EXPECT_EQ(stats.read_bytes, first.file_size - 16); // Empty record not yet read. + EXPECT_GT(stats.read_time_ns, 0); + EXPECT_EQ(stats.read_rows, 0); // Anchor rows are already counted by the scanner. + + ASSERT_TRUE(reader.read_blob(*row_positions({1, 2, 3}), 100, context, &result).ok()); + EXPECT_EQ(stats.read_calls, 7); // Empty record, then second file's footer/index/record. + EXPECT_EQ(stats.read_bytes, first.file_size + second.file_size); + expect_inline(*result, 0, ""); + expect_inline(*result, 1, std::nullopt); + expect_inline(*result, 2, "world"); +} + +TEST_F(PaimonBlobColumnReaderTest, DescriptorAccountsForMetadataWithoutReadingPayload) { + auto file = write_blob("descriptor.blob", 7, 100, {"hello", "", std::nullopt}); + io::FileReaderStats stats; + _io_context.file_reader_stats = &stats; + auto context = read_context(); + PaimonBlobColumnReader reader(blob_column("image", 7), {&file}, true); + MutableColumnPtr result; + ASSERT_TRUE(reader.read_blob(*row_positions({0, 1, 2}), 100, context, &result).ok()); + EXPECT_EQ(stats.read_calls, 2); + EXPECT_EQ(stats.read_bytes, file.file_size - (5 + 16) - 16); // Footer and index only. + EXPECT_GT(stats.read_time_ns, 0); + ASSERT_TRUE(reader.read_blob(*row_positions({0}), 100, context, &result).ok()); + EXPECT_EQ(stats.read_calls, 2); // Decoded index is reused across batches. + expect_descriptor(*result, 0, file.path, 4, 5); +} + TEST_F(PaimonBlobColumnReaderTest, SortsFilesAndRequiresContiguousAnchorCoverage) { auto first = write_blob("first.blob", 7, 1000, {"a", "b", "c"}); auto second = write_blob("second.blob", 7, 1003, {"d", "e"}); diff --git a/be/test/format_v2/blob/paimon_blob_materializer_test.cpp b/be/test/format_v2/blob/paimon_blob_materializer_test.cpp index 6ddc72e3e8be3a..a57aa67dc61726 100644 --- a/be/test/format_v2/blob/paimon_blob_materializer_test.cpp +++ b/be/test/format_v2/blob/paimon_blob_materializer_test.cpp @@ -33,6 +33,29 @@ namespace { class PaimonBlobMaterializerTest : public blob_test::PaimonBlobTest {}; +TEST_F(PaimonBlobMaterializerTest, ProfilesMaterializationAcrossSplits) { + RuntimeProfile profile("PaimonBlobTest"); + PaimonBlobMaterializer materializer; + ASSERT_TRUE(materializer.init({}, &profile).ok()); + const std::vector columns {blob_column("image", 7)}; + auto context = read_context(); + context.profile = &profile; + auto* timer = profile.get_counter("PaimonBlobMaterializeTime"); + ASSERT_NE(timer, nullptr); + EXPECT_EQ(timer->value(), 0); + for (int split = 0; split < 2; ++split) { + auto file = write_blob("split-" + std::to_string(split) + ".blob", 7, 0, {"hello"}); + ASSERT_TRUE(materializer.prepare_split(blob_split(0, 1, {file}), columns).ok()); + auto block = output_block(columns, 1); + const auto before = timer->value(); + ASSERT_TRUE(materializer.materialize(*row_positions({0}), &block, context).ok()); + EXPECT_GT(timer->value(), before); + expect_inline(*block.get_by_position(0).column, 0, "hello"); + } + materializer.reset(); + EXPECT_GT(timer->value(), 0); +} + TEST_F(PaimonBlobMaterializerTest, MatchesFieldIdsAfterRenameAndProjectionReordering) { auto image_first = write_blob("image_first.blob", 7, 1000, {"image-0", std::nullopt}); auto image_second = write_blob("image_second.blob", 7, 1002, {"", "image-3"}); diff --git a/be/test/format_v2/blob/paimon_blob_test_util.h b/be/test/format_v2/blob/paimon_blob_test_util.h index 18ae9dbf3b22a1..66c42d67b599d5 100644 --- a/be/test/format_v2/blob/paimon_blob_test_util.h +++ b/be/test/format_v2/blob/paimon_blob_test_util.h @@ -89,6 +89,7 @@ class PaimonBlobTest : public ::testing::Test { return {.system_properties = _system_properties, .runtime_state = nullptr, .io_ctx = &_io_context, + .profile = nullptr, .fs_name = _fs_name, .file_cache_admission = false, .blob_max_size = max_size}; diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index 1c1bbc093b98e7..9a5191da48e3aa 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -269,7 +269,7 @@ public boolean isConst() { } /** - * Release columns and meta information + * Release buffers owned by a writable column. Readable columns borrow BE memory. */ public void close() { if (childColumns != null) { @@ -290,9 +290,8 @@ public void close() { if (data != 0) { if (columnType.isVarbinaryType()) { freeVarbinaryData(); - } else { - OffHeap.freeMemory(data); } + OffHeap.freeMemory(data); } if (offsets != 0) { diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index aa60858e6d1d79..0a176faddabbca 100644 --- a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -38,6 +38,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; import java.math.BigDecimal; import java.math.BigInteger; import java.time.LocalDate; @@ -321,14 +325,27 @@ public FileValue getFile() { private byte[] readBlobPayload(Blob blob) { if (blob instanceof BlobRef) { // Managed .blob references carry their length, so reject before opening the payload. - // External references can have length -1 and are checked after reading below. checkBlobSize(blob.toDescriptor().length()); } - // TODO: Use bounded streaming reads for unknown-length BLOBs so the size limit can be - // enforced before toData() allocates and reads the entire payload. - byte[] payload = blob.toData(); - checkBlobSize(payload.length); - return payload; + // External descriptors can have length -1. Read at most limit + 1 bytes, + // checking before retaining each chunk instead of materializing the whole object. + try (InputStream input = blob.newInputStream(); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + byte[] buffer = new byte[(int) Math.min(8192L, blobMaxSize + 1)]; + long total = 0; + while (true) { + int request = (int) Math.min(buffer.length, blobMaxSize - total + 1); + int read = input.read(buffer, 0, request); + if (read < 0) { + return output.toByteArray(); + } + total += read; + checkBlobSize(total); + output.write(buffer, 0, read); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to read Paimon BLOB column '" + dorisType.getName() + "'", e); + } } private void checkBlobSize(long size) {