diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql index 2437a454024b01..b360e5fe13f779 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql @@ -115,6 +115,28 @@ create table blob_size_limit ( insert into blob_size_limit select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (1, 2, 11, 12, 13) order by id; +-- Real payloads at and above Doris' default 10 MiB inline limit. Keep them separate +-- from the small fixtures so ordinary payload scans do not fail with the default limit. +-- Store an MD5 instead of a hex copy of each multi-megabyte payload. +drop table if exists blob_large_payload; +create table blob_large_payload ( + id int, case_name string, image binary, expected_size int, expected_md5 string +) using paimon tblproperties ( + 'file.format' = 'parquet', 'row-tracking.enabled' = 'true', + 'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true' +); +insert into blob_large_payload +select /*+ COALESCE(1) */ id, case_name, image, expected_size, md5(image) +from ( + select id, case_name, encode(repeat('A', expected_size), 'UTF-8') as image, expected_size + from values + (1, 'limit_10m_exact', 10485760), + (2, 'limit_10m_plus_one', 10485761), + (3, 'payload_11m', 11534336) + as seed(id, case_name, expected_size) +) payloads +order by id; + -- Two BLOB fields separated by an ordinary column. NULL maps and sizes are independent. create or replace temporary view paimon_blob_pair_seed as select id, image, note, thumbnail, diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out b/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out index b631991409dc09..75d217106fb6d2 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out @@ -5,6 +5,7 @@ blob_basic blob_compacted blob_descriptor_default blob_empty +blob_large_payload blob_multi_columns blob_multi_snapshot blob_orc_fallback @@ -246,6 +247,32 @@ thumbnail 3781 0 2 0 true 10 11 true +-- !large_default_exact -- +1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true + +-- !large_default_descriptor -- +1 10485760 true true true true +2 10485761 true true true true +3 11534336 true true true true + +-- !large_raised_content -- +1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true +2 10485761 126c1549a4e98d4e0254c84619963624 true true +3 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b true true + +-- !large_metadata_only -- +1 limit_10m_exact 10485760 8b8378787c0925f42ccb829f6cc2fb97 +2 limit_10m_plus_one 10485761 126c1549a4e98d4e0254c84619963624 +3 payload_11m 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b + +-- !large_count -- +3 + +-- !large_descriptor_with_low_limit -- +1 10485760 true true true true +2 10485761 true true true true +3 11534336 true true true true + -- !limit_allowed -- 1 \N \N 2 0 diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out b/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out index b631991409dc09..75d217106fb6d2 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out @@ -5,6 +5,7 @@ blob_basic blob_compacted blob_descriptor_default blob_empty +blob_large_payload blob_multi_columns blob_multi_snapshot blob_orc_fallback @@ -246,6 +247,32 @@ thumbnail 3781 0 2 0 true 10 11 true +-- !large_default_exact -- +1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true + +-- !large_default_descriptor -- +1 10485760 true true true true +2 10485761 true true true true +3 11534336 true true true true + +-- !large_raised_content -- +1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true +2 10485761 126c1549a4e98d4e0254c84619963624 true true +3 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b true true + +-- !large_metadata_only -- +1 limit_10m_exact 10485760 8b8378787c0925f42ccb829f6cc2fb97 +2 limit_10m_plus_one 10485761 126c1549a4e98d4e0254c84619963624 +3 payload_11m 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b + +-- !large_count -- +3 + +-- !large_descriptor_with_low_limit -- +1 10485760 true true true true +2 10485761 true true true true +3 11534336 true true true true + -- !limit_allowed -- 1 \N \N 2 0 diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy index d37cb27a7440d7..60624e4d96f141 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy @@ -51,7 +51,7 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker sql "set force_jni_scanner = true" sql "set paimon_blob_max_size = 10485760" - ["blob_basic", "blob_all_null", "blob_size_limit", "blob_multi_columns", + ["blob_basic", "blob_all_null", "blob_size_limit", "blob_large_payload", "blob_multi_columns", "blob_partitioned", "blob_multi_snapshot", "blob_rollover", "blob_schema_evolution", "blob_compacted", "blob_descriptor_default"].each { table -> explain { @@ -374,7 +374,71 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker from blob_descriptor_default order by id """ - // Small payloads exercise the limit: do not allocate 10 MiB just to test an error. + // Real payloads exercise the default 10 MiB boundary without returning large values. + qt_large_default_exact """ + select id, length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)), + length(fl_get_binary(image)) <=> expected_size, + md5(cast(fl_get_binary(image) as string)) <=> expected_md5 + from blob_large_payload where id = 1 + """ + test { + sql "select length(fl_get_binary(image)) from blob_large_payload where id = 2" + exception "paimon_blob_max_size=10485760" + } + test { + sql "select length(fl_get_binary(image)) from blob_large_payload where id = 3" + exception "paimon_blob_max_size=10485760" + } + test { + sql """ + select id from blob_large_payload + where length(fl_get_binary(image)) <= 10485760 + """ + exception "paimon_blob_max_size=10485760" + } + qt_large_default_descriptor """ + select id, element_at(image, 'size'), + element_at(image, 'size') <=> expected_size, + element_at(image, 'uri') is not null, + element_at(image, 'offset') >= 4, + fl_get_binary(image) is null + from blob_large_payload@options('blob-as-descriptor'='true') order by id + """ + + sql "set paimon_blob_max_size = 12582912" + qt_large_raised_content """ + select id, length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)), + length(fl_get_binary(image)) <=> expected_size, + md5(cast(fl_get_binary(image) as string)) <=> expected_md5 + from blob_large_payload order by id + """ + + // Neither ordinary-column projection nor descriptor output needs payload bytes. + sql "set paimon_blob_max_size = 1" + qt_large_metadata_only """ + select id, case_name, expected_size, expected_md5 + from blob_large_payload order by id + """ + qt_large_count "select count(*) from blob_large_payload" + qt_large_descriptor_with_low_limit """ + select id, element_at(image, 'size'), + element_at(image, 'size') <=> expected_size, + element_at(image, 'uri') is not null, + element_at(image, 'offset') >= 4, + fl_get_binary(image) is null + from blob_large_payload@options('blob-as-descriptor'='true') order by id + """ + + // A successful inline read must not cache the larger limit for later queries. + sql "set paimon_blob_max_size = 10485760" + test { + sql "select length(fl_get_binary(image)) from blob_large_payload where id = 2" + exception "paimon_blob_max_size=10485760" + } + + // Keep the smaller boundary cases for NULL, empty and multiple BLOB columns. sql "set paimon_blob_max_size = 8" qt_limit_allowed """ select id, length(fl_get_binary(image)), diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy index d18fc48329d7c8..36606cc4cedb0c 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy @@ -51,7 +51,7 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc sql "set force_jni_scanner = false" sql "set paimon_blob_max_size = 10485760" - ["blob_basic", "blob_all_null", "blob_size_limit", "blob_multi_columns", + ["blob_basic", "blob_all_null", "blob_size_limit", "blob_large_payload", "blob_multi_columns", "blob_partitioned", "blob_multi_snapshot", "blob_rollover", "blob_schema_evolution", "blob_compacted", "blob_descriptor_default"].each { table -> explain { @@ -374,7 +374,71 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc from blob_descriptor_default order by id """ - // Small payloads exercise the limit: do not allocate 10 MiB just to test an error. + // Real payloads exercise the default 10 MiB boundary without returning large values. + qt_large_default_exact """ + select id, length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)), + length(fl_get_binary(image)) <=> expected_size, + md5(cast(fl_get_binary(image) as string)) <=> expected_md5 + from blob_large_payload where id = 1 + """ + test { + sql "select length(fl_get_binary(image)) from blob_large_payload where id = 2" + exception "paimon_blob_max_size=10485760" + } + test { + sql "select length(fl_get_binary(image)) from blob_large_payload where id = 3" + exception "paimon_blob_max_size=10485760" + } + test { + sql """ + select id from blob_large_payload + where length(fl_get_binary(image)) <= 10485760 + """ + exception "paimon_blob_max_size=10485760" + } + qt_large_default_descriptor """ + select id, element_at(image, 'size'), + element_at(image, 'size') <=> expected_size, + element_at(image, 'uri') is not null, + element_at(image, 'offset') >= 4, + fl_get_binary(image) is null + from blob_large_payload@options('blob-as-descriptor'='true') order by id + """ + + sql "set paimon_blob_max_size = 12582912" + qt_large_raised_content """ + select id, length(fl_get_binary(image)), + md5(cast(fl_get_binary(image) as string)), + length(fl_get_binary(image)) <=> expected_size, + md5(cast(fl_get_binary(image) as string)) <=> expected_md5 + from blob_large_payload order by id + """ + + // Neither ordinary-column projection nor descriptor output needs payload bytes. + sql "set paimon_blob_max_size = 1" + qt_large_metadata_only """ + select id, case_name, expected_size, expected_md5 + from blob_large_payload order by id + """ + qt_large_count "select count(*) from blob_large_payload" + qt_large_descriptor_with_low_limit """ + select id, element_at(image, 'size'), + element_at(image, 'size') <=> expected_size, + element_at(image, 'uri') is not null, + element_at(image, 'offset') >= 4, + fl_get_binary(image) is null + from blob_large_payload@options('blob-as-descriptor'='true') order by id + """ + + // A successful inline read must not cache the larger limit for later queries. + sql "set paimon_blob_max_size = 10485760" + test { + sql "select length(fl_get_binary(image)) from blob_large_payload where id = 2" + exception "paimon_blob_max_size=10485760" + } + + // Keep the smaller boundary cases for NULL, empty and multiple BLOB columns. sql "set paimon_blob_max_size = 8" qt_limit_allowed """ select id, length(fl_get_binary(image)),