From 0f19158a8d23aaff8721d946aab91daefbe76dfd Mon Sep 17 00:00:00 2001 From: pedrobtz Date: Thu, 22 Oct 2026 00:03:56 +0200 Subject: [PATCH] fix: Validate run ends, list view offsets and dictionary indices in unaligned buffers safely Default and full validation read these through aligned pointers, which is undefined behaviour for the unaligned buffers the C Data interface allows. Follows the fix for offsets buffers in #946. Co-Authored-By: Claude Opus 5.5 --- src/nanoarrow/common/array.c | 61 ++++++++++++++--- src/nanoarrow/common/array_test.cc | 102 +++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 9 deletions(-) diff --git a/src/nanoarrow/common/array.c b/src/nanoarrow/common/array.c index e4538955c..70dc6fe8d 100644 --- a/src/nanoarrow/common/array.c +++ b/src/nanoarrow/common/array.c @@ -1818,6 +1818,49 @@ static int ArrowArrayViewValidateMinimal(struct ArrowArrayView* array_view, return NANOARROW_OK; } +// Like ArrowArrayViewGetIntUnsafe() for the integer types, but without assuming +// that the data buffer is aligned to the element size: the C Data interface +// allows unaligned buffers +static int64_t ArrowArrayViewGetIntUnaligned(const struct ArrowArrayView* array_view, + int64_t i) { + const uint8_t* data = array_view->buffer_views[1].data.as_uint8; + i += array_view->offset; + switch (array_view->storage_type) { + case NANOARROW_TYPE_INT64: { + int64_t value; + memcpy(&value, data + i * sizeof(value), sizeof(value)); + return value; + } + case NANOARROW_TYPE_UINT64: { + uint64_t value; + memcpy(&value, data + i * sizeof(value), sizeof(value)); + return (int64_t)value; + } + case NANOARROW_TYPE_INT32: { + int32_t value; + memcpy(&value, data + i * sizeof(value), sizeof(value)); + return value; + } + case NANOARROW_TYPE_UINT32: { + uint32_t value; + memcpy(&value, data + i * sizeof(value), sizeof(value)); + return value; + } + case NANOARROW_TYPE_INT16: { + int16_t value; + memcpy(&value, data + i * sizeof(value), sizeof(value)); + return value; + } + case NANOARROW_TYPE_UINT16: { + uint16_t value; + memcpy(&value, data + i * sizeof(value), sizeof(value)); + return value; + } + default: + return ArrowArrayViewGetIntUnsafe(array_view, i - array_view->offset); + } +} + static int ArrowArrayViewValidateDefault(struct ArrowArrayView* array_view, struct ArrowError* error) { // Perform minimal validation. This will validate or assign @@ -1988,7 +2031,7 @@ static int ArrowArrayViewValidateDefault(struct ArrowArrayView* array_view, break; } - int64_t first_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, 0); + int64_t first_run_end = ArrowArrayViewGetIntUnaligned(run_ends_view, 0); if (first_run_end < 1) { ArrowErrorSet( error, @@ -1999,7 +2042,7 @@ static int ArrowArrayViewValidateDefault(struct ArrowArrayView* array_view, // offset + length < INT64_MAX is checked in ArrowArrayViewValidateMinimal() int64_t last_run_end = - ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1); + ArrowArrayViewGetIntUnaligned(run_ends_view, run_ends_view->length - 1); if (last_run_end < offset_plus_length) { ArrowErrorSet(error, "Last run end is %" PRId64 " but it should be >= (%" PRId64 @@ -2204,9 +2247,9 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, if (array_view->storage_type == NANOARROW_TYPE_RUN_END_ENCODED) { struct ArrowArrayView* run_ends_view = array_view->children[0]; if (run_ends_view->length > 0) { - int64_t last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, 0); + int64_t last_run_end = ArrowArrayViewGetIntUnaligned(run_ends_view, 0); for (int64_t i = 1; i < run_ends_view->length; i++) { - const int64_t run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, i); + const int64_t run_end = ArrowArrayViewGetIntUnaligned(run_ends_view, i); if (run_end <= last_run_end) { ArrowErrorSet( error, @@ -2233,11 +2276,11 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, i++) { int64_t offset, size; if (array_view->storage_type == NANOARROW_TYPE_LIST_VIEW) { - offset = offsets.data.as_int32[i]; - size = sizes.data.as_int32[i]; + offset = ArrowBufferViewGetInt32Unsafe(&offsets, i); + size = ArrowBufferViewGetInt32Unsafe(&sizes, i); } else { - offset = offsets.data.as_int64[i]; - size = sizes.data.as_int64[i]; + offset = ArrowBufferViewGetInt64Unsafe(&offsets, i); + size = ArrowBufferViewGetInt64Unsafe(&sizes, i); } if (offset < 0) { @@ -2278,7 +2321,7 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, continue; } - int64_t index = ArrowArrayViewGetIntUnsafe(array_view, i); + int64_t index = ArrowArrayViewGetIntUnaligned(array_view, i); if (index < 0 || index >= dictionary_length) { ArrowErrorSet(error, "[%" PRId64 "] Expected dictionary index >= 0 and < %" PRId64 diff --git a/src/nanoarrow/common/array_test.cc b/src/nanoarrow/common/array_test.cc index 9324cdde6..941a1c06d 100644 --- a/src/nanoarrow/common/array_test.cc +++ b/src/nanoarrow/common/array_test.cc @@ -2839,6 +2839,108 @@ TEST(ArrayTest, ArrayViewTestMove) { ArrowArrayViewReset(&array_view2); } +TEST(ArrayTest, ArrayViewTestValidateFullUnalignedIntegers) { + // Run ends, list view offsets and sizes, and dictionary indices may be in + // unaligned buffers; validation must not read them through aligned pointers + struct ArrowError error; + const int32_t values[] = {7, 8, 9}; + + { + // Dictionary indices + nanoarrow::UniqueSchema schema; + ASSERT_EQ(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_INT32), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaAllocateDictionary(schema.get()), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaInitFromType(schema->dictionary, NANOARROW_TYPE_INT32), + NANOARROW_OK); + nanoarrow::UniqueArrayView array_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error), + NANOARROW_OK); + + uint8_t indices[1 + 2 * sizeof(int32_t)] = {}; + const int32_t expected[] = {0, 2}; + memcpy(indices + 1, expected, sizeof(expected)); + array_view->length = 2; + array_view->buffer_views[1].data.as_uint8 = indices + 1; + array_view->buffer_views[1].size_bytes = sizeof(expected); + array_view->dictionary->length = 3; + array_view->dictionary->buffer_views[1].data.as_int32 = values; + array_view->dictionary->buffer_views[1].size_bytes = sizeof(values); + + EXPECT_EQ( + ArrowArrayViewValidate(array_view.get(), NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + } + + { + // Run ends + nanoarrow::UniqueSchema schema; + ArrowSchemaInit(schema.get()); + ASSERT_EQ(ArrowSchemaSetTypeRunEndEncoded(schema.get(), NANOARROW_TYPE_INT32), + NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_INT32), + NANOARROW_OK); + nanoarrow::UniqueArrayView array_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error), + NANOARROW_OK); + + uint8_t run_ends[1 + 2 * sizeof(int32_t)] = {}; + const int32_t expected[] = {1, 3}; + memcpy(run_ends + 1, expected, sizeof(expected)); + array_view->length = 3; + array_view->children[0]->length = 2; + array_view->children[0]->buffer_views[1].data.as_uint8 = run_ends + 1; + array_view->children[0]->buffer_views[1].size_bytes = sizeof(expected); + array_view->children[1]->length = 2; + array_view->children[1]->buffer_views[1].data.as_int32 = values; + array_view->children[1]->buffer_views[1].size_bytes = 2 * sizeof(int32_t); + + EXPECT_EQ( + ArrowArrayViewValidate(array_view.get(), NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << error.message; + } + + for (enum ArrowType type : {NANOARROW_TYPE_LIST_VIEW, NANOARROW_TYPE_LARGE_LIST_VIEW}) { + // List view offsets and sizes + nanoarrow::UniqueSchema schema; + ASSERT_EQ(ArrowSchemaInitFromType(schema.get(), type), NANOARROW_OK); + ASSERT_EQ(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_INT32), + NANOARROW_OK); + nanoarrow::UniqueArrayView array_view; + ASSERT_EQ(ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error), + NANOARROW_OK); + + uint8_t offsets[1 + 2 * sizeof(int64_t)] = {}; + uint8_t sizes[1 + 2 * sizeof(int64_t)] = {}; + int64_t width = type == NANOARROW_TYPE_LIST_VIEW ? 4 : 8; + if (type == NANOARROW_TYPE_LIST_VIEW) { + const int32_t expected_offsets[] = {0, 1}; + const int32_t expected_sizes[] = {1, 2}; + memcpy(offsets + 1, expected_offsets, sizeof(expected_offsets)); + memcpy(sizes + 1, expected_sizes, sizeof(expected_sizes)); + } else { + const int64_t expected_offsets[] = {0, 1}; + const int64_t expected_sizes[] = {1, 2}; + memcpy(offsets + 1, expected_offsets, sizeof(expected_offsets)); + memcpy(sizes + 1, expected_sizes, sizeof(expected_sizes)); + } + array_view->length = 2; + array_view->buffer_views[1].data.as_uint8 = offsets + 1; + array_view->buffer_views[1].size_bytes = 2 * width; + array_view->buffer_views[2].data.as_uint8 = sizes + 1; + array_view->buffer_views[2].size_bytes = 2 * width; + array_view->children[0]->length = 3; + array_view->children[0]->buffer_views[1].data.as_int32 = values; + array_view->children[0]->buffer_views[1].size_bytes = sizeof(values); + + EXPECT_EQ( + ArrowArrayViewValidate(array_view.get(), NANOARROW_VALIDATION_LEVEL_FULL, &error), + NANOARROW_OK) + << ArrowTypeString(type) << ": " << error.message; + } +} + TEST(ArrayTest, ArrayViewTestString) { struct ArrowArrayView array_view; struct ArrowError error;