Skip to content

fix: Validate run ends, list view offsets and dictionary indices in unaligned buffers safely #990

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pedrobtz wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: Validate run ends, list view offsets and dictionary indices in unaligned buffers safely #990

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 52 additions & 9 deletions src/nanoarrow/common/array.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1818,6 +1818,49 @@ static int ArrowArrayViewValidateMinimal(struct ArrowArrayView* array_view,
return NANOARROW_OK;
}

// Like ArrowArrayViewGetIntUnsafe() for the integer types, but without assuming
// that the data buffer is aligned to the element size: the C Data interface
// allows unaligned buffers
static int64_t ArrowArrayViewGetIntUnaligned(const struct ArrowArrayView* array_view,
int64_t i) {
const uint8_t* data = array_view->buffer_views[1].data.as_uint8;
i += array_view->offset;
switch (array_view->storage_type) {
case NANOARROW_TYPE_INT64: {
int64_t value;
memcpy(&value, data + i * sizeof(value), sizeof(value));
return value;
}
case NANOARROW_TYPE_UINT64: {
uint64_t value;
memcpy(&value, data + i * sizeof(value), sizeof(value));
return (int64_t)value;
}
case NANOARROW_TYPE_INT32: {
int32_t value;
memcpy(&value, data + i * sizeof(value), sizeof(value));
return value;
}
case NANOARROW_TYPE_UINT32: {
uint32_t value;
memcpy(&value, data + i * sizeof(value), sizeof(value));
return value;
}
case NANOARROW_TYPE_INT16: {
int16_t value;
memcpy(&value, data + i * sizeof(value), sizeof(value));
return value;
}
case NANOARROW_TYPE_UINT16: {
uint16_t value;
memcpy(&value, data + i * sizeof(value), sizeof(value));
return value;
}
default:
return ArrowArrayViewGetIntUnsafe(array_view, i - array_view->offset);
}
}

static int ArrowArrayViewValidateDefault(struct ArrowArrayView* array_view,
struct ArrowError* error) {
// Perform minimal validation. This will validate or assign
Expand Down Expand Up @@ -1988,7 +2031,7 @@ static int ArrowArrayViewValidateDefault(struct ArrowArrayView* array_view,
break;
}

int64_t first_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, 0);
int64_t first_run_end = ArrowArrayViewGetIntUnaligned(run_ends_view, 0);
if (first_run_end < 1) {
ArrowErrorSet(
error,
Expand All @@ -1999,7 +2042,7 @@ static int ArrowArrayViewValidateDefault(struct ArrowArrayView* array_view,

// offset + length < INT64_MAX is checked in ArrowArrayViewValidateMinimal()
int64_t last_run_end =
ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1);
ArrowArrayViewGetIntUnaligned(run_ends_view, run_ends_view->length - 1);
if (last_run_end < offset_plus_length) {
ArrowErrorSet(error,
"Last run end is %" PRId64 " but it should be >= (%" PRId64
Expand Down Expand Up @@ -2204,9 +2247,9 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view,
if (array_view->storage_type == NANOARROW_TYPE_RUN_END_ENCODED) {
struct ArrowArrayView* run_ends_view = array_view->children[0];
if (run_ends_view->length > 0) {
int64_t last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, 0);
int64_t last_run_end = ArrowArrayViewGetIntUnaligned(run_ends_view, 0);
for (int64_t i = 1; i < run_ends_view->length; i++) {
const int64_t run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, i);
const int64_t run_end = ArrowArrayViewGetIntUnaligned(run_ends_view, i);
if (run_end <= last_run_end) {
ArrowErrorSet(
error,
Expand All @@ -2233,11 +2276,11 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view,
i++) {
int64_t offset, size;
if (array_view->storage_type == NANOARROW_TYPE_LIST_VIEW) {
offset = offsets.data.as_int32[i];
size = sizes.data.as_int32[i];
offset = ArrowBufferViewGetInt32Unsafe(&offsets, i);
size = ArrowBufferViewGetInt32Unsafe(&sizes, i);
} else {
offset = offsets.data.as_int64[i];
size = sizes.data.as_int64[i];
offset = ArrowBufferViewGetInt64Unsafe(&offsets, i);
size = ArrowBufferViewGetInt64Unsafe(&sizes, i);
}

if (offset < 0) {
Expand Down Expand Up @@ -2278,7 +2321,7 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view,
continue;
}

int64_t index = ArrowArrayViewGetIntUnsafe(array_view, i);
int64_t index = ArrowArrayViewGetIntUnaligned(array_view, i);
if (index < 0 || index >= dictionary_length) {
ArrowErrorSet(error,
"[%" PRId64 "] Expected dictionary index >= 0 and < %" PRId64
Expand Down
102 changes: 102 additions & 0 deletions src/nanoarrow/common/array_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2839,6 +2839,108 @@ TEST(ArrayTest, ArrayViewTestMove) {
ArrowArrayViewReset(&array_view2);
}

TEST(ArrayTest, ArrayViewTestValidateFullUnalignedIntegers) {
// Run ends, list view offsets and sizes, and dictionary indices may be in
// unaligned buffers; validation must not read them through aligned pointers
struct ArrowError error;
const int32_t values[] = {7, 8, 9};

{
// Dictionary indices
nanoarrow::UniqueSchema schema;
ASSERT_EQ(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_INT32), NANOARROW_OK);
ASSERT_EQ(ArrowSchemaAllocateDictionary(schema.get()), NANOARROW_OK);
ASSERT_EQ(ArrowSchemaInitFromType(schema->dictionary, NANOARROW_TYPE_INT32),
NANOARROW_OK);
nanoarrow::UniqueArrayView array_view;
ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error),
NANOARROW_OK);

uint8_t indices[1 + 2 * sizeof(int32_t)] = {};
const int32_t expected[] = {0, 2};
memcpy(indices + 1, expected, sizeof(expected));
array_view->length = 2;
array_view->buffer_views[1].data.as_uint8 = indices + 1;
array_view->buffer_views[1].size_bytes = sizeof(expected);
array_view->dictionary->length = 3;
array_view->dictionary->buffer_views[1].data.as_int32 = values;
array_view->dictionary->buffer_views[1].size_bytes = sizeof(values);

EXPECT_EQ(
ArrowArrayViewValidate(array_view.get(), NANOARROW_VALIDATION_LEVEL_FULL, &error),
NANOARROW_OK)
<< error.message;
}

{
// Run ends
nanoarrow::UniqueSchema schema;
ArrowSchemaInit(schema.get());
ASSERT_EQ(ArrowSchemaSetTypeRunEndEncoded(schema.get(), NANOARROW_TYPE_INT32),
NANOARROW_OK);
ASSERT_EQ(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_INT32),
NANOARROW_OK);
nanoarrow::UniqueArrayView array_view;
ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error),
NANOARROW_OK);

uint8_t run_ends[1 + 2 * sizeof(int32_t)] = {};
const int32_t expected[] = {1, 3};
memcpy(run_ends + 1, expected, sizeof(expected));
array_view->length = 3;
array_view->children[0]->length = 2;
array_view->children[0]->buffer_views[1].data.as_uint8 = run_ends + 1;
array_view->children[0]->buffer_views[1].size_bytes = sizeof(expected);
array_view->children[1]->length = 2;
array_view->children[1]->buffer_views[1].data.as_int32 = values;
array_view->children[1]->buffer_views[1].size_bytes = 2 * sizeof(int32_t);

EXPECT_EQ(
ArrowArrayViewValidate(array_view.get(), NANOARROW_VALIDATION_LEVEL_FULL, &error),
NANOARROW_OK)
<< error.message;
}

for (enum ArrowType type : {NANOARROW_TYPE_LIST_VIEW, NANOARROW_TYPE_LARGE_LIST_VIEW}) {
// List view offsets and sizes
nanoarrow::UniqueSchema schema;
ASSERT_EQ(ArrowSchemaInitFromType(schema.get(), type), NANOARROW_OK);
ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_INT32),
NANOARROW_OK);
nanoarrow::UniqueArrayView array_view;
ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error),
NANOARROW_OK);

uint8_t offsets[1 + 2 * sizeof(int64_t)] = {};
uint8_t sizes[1 + 2 * sizeof(int64_t)] = {};
int64_t width = type == NANOARROW_TYPE_LIST_VIEW ? 4 : 8;
if (type == NANOARROW_TYPE_LIST_VIEW) {
const int32_t expected_offsets[] = {0, 1};
const int32_t expected_sizes[] = {1, 2};
memcpy(offsets + 1, expected_offsets, sizeof(expected_offsets));
memcpy(sizes + 1, expected_sizes, sizeof(expected_sizes));
} else {
const int64_t expected_offsets[] = {0, 1};
const int64_t expected_sizes[] = {1, 2};
memcpy(offsets + 1, expected_offsets, sizeof(expected_offsets));
memcpy(sizes + 1, expected_sizes, sizeof(expected_sizes));
}
array_view->length = 2;
array_view->buffer_views[1].data.as_uint8 = offsets + 1;
array_view->buffer_views[1].size_bytes = 2 * width;
array_view->buffer_views[2].data.as_uint8 = sizes + 1;
array_view->buffer_views[2].size_bytes = 2 * width;
array_view->children[0]->length = 3;
array_view->children[0]->buffer_views[1].data.as_int32 = values;
array_view->children[0]->buffer_views[1].size_bytes = sizeof(values);

EXPECT_EQ(
ArrowArrayViewValidate(array_view.get(), NANOARROW_VALIDATION_LEVEL_FULL, &error),
NANOARROW_OK)
<< ArrowTypeString(type) << ": " << error.message;
}
}

TEST(ArrayTest, ArrayViewTestString) {
struct ArrowArrayView array_view;
struct ArrowError error;
Expand Down