Skip to content

GH-50472: [C++][Gandiva] Guard mask length overflow and truncated UTF-8 #50947

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
VrtxOmega wants to merge 3 commits into
base: main
Choose a base branch
from
Open

GH-50472: [C++][Gandiva] Guard mask length overflow and truncated UTF-8 #50947

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e4905af
Fix integer overflow in mask_utf8_utf8_utf8_utf8
VrtxOmega Aug 21, 2026
58d9cbb
Fix out-of-bounds read in upper/lower/initcap on truncated UTF-8
VrtxOmega Aug 21, 2026
dd47db3
Test Gandiva mask overflow and truncated UTF-8 conversion
VrtxOmega Sep 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions cpp/src/gandiva/gdv_function_stubs.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "arrow/util/base64.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/double_conversion_internal.h"
#include "arrow/util/int_util_overflow.h"
#include "arrow/util/value_parsing.h"

#include "gandiva/encrypt_utils.h"
Expand Down Expand Up @@ -663,8 +664,14 @@ const char* mask_utf8_utf8_utf8_utf8(int64_t context, const char* data, int32_t
return nullptr;
}

int32_t max_length =
std::max(upper_length, std::max(lower_length, num_length)) * data_len;
int32_t max_length;
if (ARROW_PREDICT_FALSE(arrow::internal::MultiplyWithOverflow(
std::max(upper_length, std::max(lower_length, num_length)), data_len,
&max_length))) {
gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
*out_len = 0;
return nullptr;
}
char* out = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, max_length));
if (out == nullptr) {
gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string");
Expand Down
40 changes: 39 additions & 1 deletion cpp/src/gandiva/gdv_function_stubs_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -805,7 +805,7 @@ TEST(TestGdvFnStubs, TestInitCap) {
EXPECT_EQ(std::string(out_str, out_len), "{Õhp,Pqśv}Ń+");
EXPECT_FALSE(ctx.has_error());

out_str = gdv_fn_initcap_utf8(ctx_ptr, "sɦasasdsɦsd\"sdsdɦ", 19, &out_len);
out_str = gdv_fn_initcap_utf8(ctx_ptr, "sɦasasdsɦsd\"sdsdɦ", 20, &out_len);
EXPECT_EQ(std::string(out_str, out_len), "Sɦasasdsɦsd\"Sdsdɦ");
EXPECT_FALSE(ctx.has_error());

Expand Down Expand Up @@ -1188,6 +1188,44 @@ TEST(TestGdvFnStubs, TestMaskTruncatedUtf8NoOverread) {
EXPECT_TRUE(ctx.has_error());
}

TEST(TestGdvFnStubs, TestMaskOutputLengthOverflow) {
gandiva::ExecutionContext ctx;
auto ctx_ptr = reinterpret_cast<int64_t>(&ctx);
int32_t out_len = -1;
const std::string data(65536, 'A');
const std::string replacement(65536, 'X');
auto result = mask_utf8_utf8_utf8_utf8(
ctx_ptr, data.data(), static_cast<int32_t>(data.size()), replacement.data(),
static_cast<int32_t>(replacement.size()), "x", 1, "n", 1, &out_len);
EXPECT_EQ(result, nullptr);
EXPECT_EQ(out_len, 0);
EXPECT_TRUE(ctx.has_error());
}

TEST(TestGdvFnStubs, TestCaseConversionTruncatedUtf8) {
const std::string inputs[] = {"a\xc2\xa2", "a\xe2\x82\xac", "a\xf0\x9f\x98\x80"};
for (auto convert : {gdv_fn_lower_utf8, gdv_fn_upper_utf8, gdv_fn_initcap_utf8}) {
for (const auto& input : inputs) {
const auto full_len = static_cast<int32_t>(input.size());
for (int32_t len = 2; len < full_len; ++len) {
gandiva::ExecutionContext ctx;
auto ctx_ptr = reinterpret_cast<int64_t>(&ctx);
int32_t out_len = -1;
// Bytes beyond len complete the glyph, but are outside the input slice.
convert(ctx_ptr, input.data(), len, &out_len);
EXPECT_EQ(out_len, 0);
EXPECT_TRUE(ctx.has_error());
}
gandiva::ExecutionContext ctx;
auto ctx_ptr = reinterpret_cast<int64_t>(&ctx);
int32_t out_len = -1;
convert(ctx_ptr, input.data(), full_len, &out_len);
EXPECT_GT(out_len, 0);
EXPECT_FALSE(ctx.has_error());
}
}
}

TEST(TestGdvFnStubs, TestTranslate) {
gandiva::ExecutionContext ctx;
int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
Expand Down
24 changes: 24 additions & 0 deletions cpp/src/gandiva/gdv_string_function_stubs.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,14 @@ const char* gdv_fn_lower_utf8(int64_t context, const char* data, int32_t data_le
}

// Control reaches here when we encounter a multibyte character
// Ensure the multibyte sequence fits within the buffer to avoid
// reading past data_len (truncated trailing multibyte sequence).
if (char_len > data_len - i) {
gdv_fn_set_error_for_invalid_utf8(context, data[i]);

@dmitry-chirkov-dremio dmitry-chirkov-dremio Oct 15, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we also add NativeFunction::kCanReturnErrors to the lower and upper registrations in function_registry_string.cc, following the same NULL-handling fix as #49035 for binary_string? initcap already has the flag.

Without it, Gandiva invokes these functions even on NULL inputs, so the new truncation checks can fail a batch on bytes belonging to a NULL slot.

Please add Projector-level regressions for both functions, following the setup in TestUtf8.TestInternalAllocs. Construct that NULL input and assert evaluation succeeds and the output remains NULL. Direct stub tests bypass the generated validity handling and cannot catch this.

*out_len = 0;
return "";
}

const auto* in_char = (const uint8_t*)(data + i);

// Decode the multibyte character
Expand Down Expand Up @@ -366,6 +374,14 @@ const char* gdv_fn_upper_utf8(int64_t context, const char* data, int32_t data_le
}

// Control reaches here when we encounter a multibyte character
// Ensure the multibyte sequence fits within the buffer to avoid
// reading past data_len (truncated trailing multibyte sequence).
if (char_len > data_len - i) {
gdv_fn_set_error_for_invalid_utf8(context, data[i]);
*out_len = 0;
return "";
}

const auto* in_char = (const uint8_t*)(data + i);

// Decode the multibyte character
Expand Down Expand Up @@ -584,6 +600,14 @@ const char* gdv_fn_initcap_utf8(int64_t context, const char* data, int32_t data_
char_len = gdv_fn_utf8_char_length(data[i]);

// Control reaches here when we encounter a multibyte character
// Ensure the multibyte sequence fits within the buffer to avoid
// reading past data_len (truncated trailing multibyte sequence).
if (char_len > data_len - i) {
gdv_fn_set_error_for_invalid_utf8(context, data[i]);
*out_len = 0;
return "";
}

const auto* in_char = (const uint8_t*)(data + i);

// Decode the multibyte character
Expand Down
Loading