Skip to content

paimon-blob-4.1: update blob large payload #69517

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zhangstar333 merged 1 commit into from
Oct 23, 2026
Merged

paimon-blob-4.1: update blob large payload #69517

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .../thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run15.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,28 @@ create table blob_size_limit (
insert into blob_size_limit
select /*+ COALESCE(1) */ * from paimon_blob_seed where id in (1, 2, 11, 12, 13) order by id;

-- Real payloads at and above Doris' default 10 MiB inline limit. Keep them separate
-- from the small fixtures so ordinary payload scans do not fail with the default limit.
-- Store an MD5 instead of a hex copy of each multi-megabyte payload.
drop table if exists blob_large_payload;
create table blob_large_payload (
id int, case_name string, image binary, expected_size int, expected_md5 string
) using paimon tblproperties (
'file.format' = 'parquet', 'row-tracking.enabled' = 'true',
'data-evolution.enabled' = 'true', 'blob-field' = 'image', 'write-only' = 'true'
);
insert into blob_large_payload
select /*+ COALESCE(1) */ id, case_name, image, expected_size, md5(image)
from (
select id, case_name, encode(repeat('A', expected_size), 'UTF-8') as image, expected_size
from values
(1, 'limit_10m_exact', 10485760),
(2, 'limit_10m_plus_one', 10485761),
(3, 'payload_11m', 11534336)
as seed(id, case_name, expected_size)
) payloads
order by id;

-- Two BLOB fields separated by an ordinary column. NULL maps and sizes are independent.
create or replace temporary view paimon_blob_pair_seed as
select id, image, note, thumbnail,
Expand Down
27 changes: 27 additions & 0 deletions regression-test/data/external_table_p0/paimon/test_paimon_blob_jni.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ blob_basic
blob_compacted
blob_descriptor_default
blob_empty
blob_large_payload
blob_multi_columns
blob_multi_snapshot
blob_orc_fallback
Expand Down Expand Up @@ -246,6 +247,32 @@ thumbnail 3781 0
2 0 true
10 11 true

-- !large_default_exact --
1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true

-- !large_default_descriptor --
1 10485760 true true true true
2 10485761 true true true true
3 11534336 true true true true

-- !large_raised_content --
1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true
2 10485761 126c1549a4e98d4e0254c84619963624 true true
3 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b true true

-- !large_metadata_only --
1 limit_10m_exact 10485760 8b8378787c0925f42ccb829f6cc2fb97
2 limit_10m_plus_one 10485761 126c1549a4e98d4e0254c84619963624
3 payload_11m 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b

-- !large_count --
3

-- !large_descriptor_with_low_limit --
1 10485760 true true true true
2 10485761 true true true true
3 11534336 true true true true

-- !limit_allowed --
1 \N \N
2 0
Expand Down
27 changes: 27 additions & 0 deletions regression-test/data/external_table_p0/paimon/test_paimon_blob_native.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ blob_basic
blob_compacted
blob_descriptor_default
blob_empty
blob_large_payload
blob_multi_columns
blob_multi_snapshot
blob_orc_fallback
Expand Down Expand Up @@ -246,6 +247,32 @@ thumbnail 3781 0
2 0 true
10 11 true

-- !large_default_exact --
1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true

-- !large_default_descriptor --
1 10485760 true true true true
2 10485761 true true true true
3 11534336 true true true true

-- !large_raised_content --
1 10485760 8b8378787c0925f42ccb829f6cc2fb97 true true
2 10485761 126c1549a4e98d4e0254c84619963624 true true
3 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b true true

-- !large_metadata_only --
1 limit_10m_exact 10485760 8b8378787c0925f42ccb829f6cc2fb97
2 limit_10m_plus_one 10485761 126c1549a4e98d4e0254c84619963624
3 payload_11m 11534336 cbbd1fd2a18bcea6f08205a0fa8baa5b

-- !large_count --
3

-- !large_descriptor_with_low_limit --
1 10485760 true true true true
2 10485761 true true true true
3 11534336 true true true true

-- !limit_allowed --
1 \N \N
2 0
Expand Down
68 changes: 66 additions & 2 deletions regression-test/suites/external_table_p0/paimon/test_paimon_blob_jni.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker
sql "set force_jni_scanner = true"
sql "set paimon_blob_max_size = 10485760"

["blob_basic", "blob_all_null", "blob_size_limit", "blob_multi_columns",
["blob_basic", "blob_all_null", "blob_size_limit", "blob_large_payload", "blob_multi_columns",
"blob_partitioned", "blob_multi_snapshot", "blob_rollover", "blob_schema_evolution",
"blob_compacted", "blob_descriptor_default"].each { table ->
explain {
Expand Down Expand Up @@ -374,7 +374,71 @@ suite("test_paimon_blob_jni", "p0,external,doris,external_docker,external_docker
from blob_descriptor_default order by id
"""

// Small payloads exercise the limit: do not allocate 10 MiB just to test an error.
// Real payloads exercise the default 10 MiB boundary without returning large values.
qt_large_default_exact """
select id, length(fl_get_binary(image)),
md5(cast(fl_get_binary(image) as string)),
length(fl_get_binary(image)) <=> expected_size,
md5(cast(fl_get_binary(image) as string)) <=> expected_md5
from blob_large_payload where id = 1
"""
test {
sql "select length(fl_get_binary(image)) from blob_large_payload where id = 2"
exception "paimon_blob_max_size=10485760"
}
test {
sql "select length(fl_get_binary(image)) from blob_large_payload where id = 3"
exception "paimon_blob_max_size=10485760"
}
test {
sql """
select id from blob_large_payload
where length(fl_get_binary(image)) <= 10485760
"""
exception "paimon_blob_max_size=10485760"
}
qt_large_default_descriptor """
select id, element_at(image, 'size'),
element_at(image, 'size') <=> expected_size,
element_at(image, 'uri') is not null,
element_at(image, 'offset') >= 4,
fl_get_binary(image) is null
from blob_large_payload@options('blob-as-descriptor'='true') order by id
"""

sql "set paimon_blob_max_size = 12582912"
qt_large_raised_content """
select id, length(fl_get_binary(image)),
md5(cast(fl_get_binary(image) as string)),
length(fl_get_binary(image)) <=> expected_size,
md5(cast(fl_get_binary(image) as string)) <=> expected_md5
from blob_large_payload order by id
"""

// Neither ordinary-column projection nor descriptor output needs payload bytes.
sql "set paimon_blob_max_size = 1"
qt_large_metadata_only """
select id, case_name, expected_size, expected_md5
from blob_large_payload order by id
"""
qt_large_count "select count(*) from blob_large_payload"
qt_large_descriptor_with_low_limit """
select id, element_at(image, 'size'),
element_at(image, 'size') <=> expected_size,
element_at(image, 'uri') is not null,
element_at(image, 'offset') >= 4,
fl_get_binary(image) is null
from blob_large_payload@options('blob-as-descriptor'='true') order by id
"""

// A successful inline read must not cache the larger limit for later queries.
sql "set paimon_blob_max_size = 10485760"
test {
sql "select length(fl_get_binary(image)) from blob_large_payload where id = 2"
exception "paimon_blob_max_size=10485760"
}

// Keep the smaller boundary cases for NULL, empty and multiple BLOB columns.
sql "set paimon_blob_max_size = 8"
qt_limit_allowed """
select id, length(fl_get_binary(image)),
Expand Down
68 changes: 66 additions & 2 deletions regression-test/suites/external_table_p0/paimon/test_paimon_blob_native.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc
sql "set force_jni_scanner = false"
sql "set paimon_blob_max_size = 10485760"

["blob_basic", "blob_all_null", "blob_size_limit", "blob_multi_columns",
["blob_basic", "blob_all_null", "blob_size_limit", "blob_large_payload", "blob_multi_columns",
"blob_partitioned", "blob_multi_snapshot", "blob_rollover", "blob_schema_evolution",
"blob_compacted", "blob_descriptor_default"].each { table ->
explain {
Expand Down Expand Up @@ -374,7 +374,71 @@ suite("test_paimon_blob_native", "p0,external,doris,external_docker,external_doc
from blob_descriptor_default order by id
"""

// Small payloads exercise the limit: do not allocate 10 MiB just to test an error.
// Real payloads exercise the default 10 MiB boundary without returning large values.
qt_large_default_exact """
select id, length(fl_get_binary(image)),
md5(cast(fl_get_binary(image) as string)),
length(fl_get_binary(image)) <=> expected_size,
md5(cast(fl_get_binary(image) as string)) <=> expected_md5
from blob_large_payload where id = 1
"""
test {
sql "select length(fl_get_binary(image)) from blob_large_payload where id = 2"
exception "paimon_blob_max_size=10485760"
}
test {
sql "select length(fl_get_binary(image)) from blob_large_payload where id = 3"
exception "paimon_blob_max_size=10485760"
}
test {
sql """
select id from blob_large_payload
where length(fl_get_binary(image)) <= 10485760
"""
exception "paimon_blob_max_size=10485760"
}
qt_large_default_descriptor """
select id, element_at(image, 'size'),
element_at(image, 'size') <=> expected_size,
element_at(image, 'uri') is not null,
element_at(image, 'offset') >= 4,
fl_get_binary(image) is null
from blob_large_payload@options('blob-as-descriptor'='true') order by id
"""

sql "set paimon_blob_max_size = 12582912"
qt_large_raised_content """
select id, length(fl_get_binary(image)),
md5(cast(fl_get_binary(image) as string)),
length(fl_get_binary(image)) <=> expected_size,
md5(cast(fl_get_binary(image) as string)) <=> expected_md5
from blob_large_payload order by id
"""

// Neither ordinary-column projection nor descriptor output needs payload bytes.
sql "set paimon_blob_max_size = 1"
qt_large_metadata_only """
select id, case_name, expected_size, expected_md5
from blob_large_payload order by id
"""
qt_large_count "select count(*) from blob_large_payload"
qt_large_descriptor_with_low_limit """
select id, element_at(image, 'size'),
element_at(image, 'size') <=> expected_size,
element_at(image, 'uri') is not null,
element_at(image, 'offset') >= 4,
fl_get_binary(image) is null
from blob_large_payload@options('blob-as-descriptor'='true') order by id
"""

// A successful inline read must not cache the larger limit for later queries.
sql "set paimon_blob_max_size = 10485760"
test {
sql "select length(fl_get_binary(image)) from blob_large_payload where id = 2"
exception "paimon_blob_max_size=10485760"
}

// Keep the smaller boundary cases for NULL, empty and multiple BLOB columns.
sql "set paimon_blob_max_size = 8"
qt_limit_allowed """
select id, length(fl_get_binary(image)),
Expand Down
Loading