From 1779f1ef3e6582e8ac662fd725ac09ff887f849c Mon Sep 17 00:00:00 2001 From: Arthur Chan Date: Tue, 20 Oct 2026 09:35:24 +0100 Subject: [PATCH] OSS-Fuzz: Add new fuzzer/corpora/dict targets arrow compute process Signed-off-by: Arthur Chan --- .../fuzzing/dicts/arrow-compute-fuzz.dict | 17 ++ .../fuzzing/generate_corpuses.sh | 6 + cpp/src/arrow/compute/CMakeLists.txt | 22 +++ cpp/src/arrow/compute/fuzz.cc | 142 +++++++++++++++ cpp/src/arrow/compute/generate_fuzz_corpus.cc | 169 ++++++++++++++++++ 5 files changed, 356 insertions(+) create mode 100644 cpp/build-support/fuzzing/dicts/arrow-compute-fuzz.dict create mode 100644 cpp/src/arrow/compute/fuzz.cc create mode 100644 cpp/src/arrow/compute/generate_fuzz_corpus.cc diff --git a/cpp/build-support/fuzzing/dicts/arrow-compute-fuzz.dict b/cpp/build-support/fuzzing/dicts/arrow-compute-fuzz.dict new file mode 100644 index 000000000000..8b2d5154b96c --- /dev/null +++ b/cpp/build-support/fuzzing/dicts/arrow-compute-fuzz.dict @@ -0,0 +1,17 @@ +# Tokens for the arrow-compute-fuzz target: Arrow IPC stream framing plus +# common flatbuffer scaffolding, to help libFuzzer reach the compute kernels. +# +"\xff\xff\xff\xff" +"ARROW1\x00\x00" +"\x00\x00\x00\x00" +"\x01\x00\x00\x00" +"\x08\x00\x00\x00\x00\x00\x0a\x00" +"\x0c\x00\x10\x00\x06\x00\x08\x00" +"\x02\x00\x05\x00" +"\x03\x00\x04\x00" +"\x02\x00\x08\x00" +"\x08\x00\x0c\x00\x00\x00\x04\x00" +"\x02\x00\x03\x00" +"\x0a\x00\x08\x00\x0c\x00\x06\x00" +"schema" +"\x64\x00\x00\x00" \ No newline at end of file diff --git a/cpp/build-support/fuzzing/generate_corpuses.sh b/cpp/build-support/fuzzing/generate_corpuses.sh index 2d95e110b516..988a113179b7 100755 --- a/cpp/build-support/fuzzing/generate_corpuses.sh +++ b/cpp/build-support/fuzzing/generate_corpuses.sh @@ -90,3 +90,9 @@ cp "${ARROW_ROOT}"/testing/data/csv/*.csv "${CORPUS_DIR}" # Add examples from Pandas test suite find "${PANDAS_DIR}/" -name "*.csv" -exec cp --backup=numbered '{}' "${CORPUS_DIR}" \; "${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-csv-fuzz_seed_corpus.zip" + +# Compute kernels + +rm -rf "${CORPUS_DIR}" +"${OUT}/arrow-compute-generate-fuzz-corpus" "${CORPUS_DIR}" +"${ARROW_CPP}/build-support/fuzzing/pack_corpus.py" "${CORPUS_DIR}" "${OUT}/arrow-compute-fuzz_seed_corpus.zip" diff --git a/cpp/src/arrow/compute/CMakeLists.txt b/cpp/src/arrow/compute/CMakeLists.txt index bfddb66358ab..804620d1366d 100644 --- a/cpp/src/arrow/compute/CMakeLists.txt +++ b/cpp/src/arrow/compute/CMakeLists.txt @@ -190,3 +190,25 @@ add_arrow_compute_benchmark(function_benchmark) add_subdirectory(kernels) add_subdirectory(row) + +if(ARROW_BUILD_FUZZING_UTILITIES) + add_executable(arrow-compute-generate-fuzz-corpus generate_fuzz_corpus.cc) + if(ARROW_BUILD_STATIC) + target_link_libraries(arrow-compute-generate-fuzz-corpus arrow_compute_static + ${ARROW_UTIL_LIB} ${ARROW_TEST_LINK_LIBS}) + else() + target_link_libraries(arrow-compute-generate-fuzz-corpus arrow_compute_shared + ${ARROW_UTIL_LIB} ${ARROW_TEST_LINK_LIBS}) + endif() +endif() + +add_arrow_fuzz_target(fuzz PREFIX "arrow-compute") + +# add_arrow_fuzz_target only links libarrow; the compute kernels (including +# compute::Initialize()) live in the separate libarrow_compute, so link it +# explicitly here. +if(ARROW_BUILD_STATIC) + target_link_libraries(arrow-compute-fuzz arrow_compute_static) +else() + target_link_libraries(arrow-compute-fuzz arrow_compute_shared) +endif() diff --git a/cpp/src/arrow/compute/fuzz.cc b/cpp/src/arrow/compute/fuzz.cc new file mode 100644 index 000000000000..a0ab86fd5c34 --- /dev/null +++ b/cpp/src/arrow/compute/fuzz.cc @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// A fuzzer for the compute kernels: read an IPC stream, then evaluate +// compute functions selected by a packed header on the decoded batches. + +#include +#include +#include +#include + +#include "arrow/buffer.h" +#include "arrow/compute/exec.h" +#include "arrow/compute/initialize.h" +#include "arrow/datum.h" +#include "arrow/io/memory.h" +#include "arrow/ipc/options.h" +#include "arrow/ipc/reader.h" +#include "arrow/record_batch.h" +#include "arrow/scalar.h" +#include "arrow/status.h" +#include "arrow/util/fuzz_internal.h" +#include "arrow/util/macros.h" + +namespace arrow { +namespace compute { + +namespace { + +constexpr uint8_t kFuzzHeaderMarker = 0xFA; +constexpr int kFuzzHeaderSize = 8; + +// Function names indexable by the header selector byte. All of these can be +// called with default options; unsupported (function, argument type) +// combinations simply return a non-OK Status, which is fine for fuzzing. +const std::vector& FuzzFunctions() { + static const std::vector functions = { + // unary scalar + "abs", "negate", "floor", "ceil", "sqrt", + // vector + "unique", "value_counts", "dictionary_encode", + // aggregate + "sum", "mean", "min_max", "approximate_median", "count", + // binary scalar + "add", "subtract", "multiply", "divide", "power", "max", "min", + }; + return functions; +} + +} // namespace + +// Fuzz input layout (same convention as the CSV fuzzer): +// [0] marker 0xFA, [1] function selector, [2] argument mode, +// [3] scalar argument byte, [4..7] reserved. +// Inputs smaller than 8 bytes or without the marker are treated as plain +// IPC streams and only validated (legacy behavior). +Status FuzzCompute(const uint8_t* data, int64_t size) { + ARROW_UNUSED(Initialize()); + + MemoryPool* pool = ::arrow::internal::fuzzing_memory_pool(); + + ipc::IpcReadOptions read_options; + read_options.use_threads = false; + read_options.memory_pool = pool; + + auto buffer = std::make_shared(data, size); + io::BufferReader buffer_reader(buffer); + ARROW_ASSIGN_OR_RAISE(auto batch_reader, + ipc::RecordBatchStreamReader::Open(&buffer_reader, + read_options)); + + uint8_t function_selector = 0; + uint8_t arg_mode = 0; + uint8_t scalar_byte = 0; + bool run_kernels = false; + if (size >= kFuzzHeaderSize && data[0] == kFuzzHeaderMarker) { + run_kernels = true; + function_selector = data[1]; + arg_mode = data[2]; + scalar_byte = data[3]; + } + + ExecContext exec_ctx(pool); + + Status st; + while (true) { + ARROW_ASSIGN_OR_RAISE(auto batch, batch_reader->ReadNext()); + if (!batch.batch && !batch.custom_metadata) { + // EOS + break; + } + if (batch.batch) { + st &= batch.batch->Validate(); + if (run_kernels) { + const auto& functions = FuzzFunctions(); + const auto& function_name = functions[function_selector % functions.size()]; + const auto& columns = batch.batch->columns(); + std::vector args; + switch (arg_mode % 3) { + case 0: + args = {Datum(columns[0])}; + break; + case 1: + args = {Datum(columns[0]), + Datum(Int64Scalar(static_cast(scalar_byte)))}; + break; + default: + args = {Datum(columns[0]), + Datum(columns[columns.size() > 1 ? 1 : 0])}; + break; + } + // Kernels are allowed to reject inputs; only crashes are interesting. + ARROW_UNUSED(CallFunction(function_name, args, /*options=*/nullptr, &exec_ctx) + .status()); + } + } + } + + return st; +} + +} // namespace compute +} // namespace arrow + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + auto status = arrow::compute::FuzzCompute(data, static_cast(size)); + return arrow::internal::LogFuzzStatus(status, data, static_cast(size)); +} \ No newline at end of file diff --git a/cpp/src/arrow/compute/generate_fuzz_corpus.cc b/cpp/src/arrow/compute/generate_fuzz_corpus.cc new file mode 100644 index 000000000000..1e3c24d7c837 --- /dev/null +++ b/cpp/src/arrow/compute/generate_fuzz_corpus.cc @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// A command line executable that generates seed corpus files for the +// arrow-compute-fuzz target: small IPC streams, each prefixed with the +// 8-byte compute-fuzz header (marker 0xFA + function/argument selectors). + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/buffer.h" +#include "arrow/io/file.h" +#include "arrow/io/memory.h" +#include "arrow/ipc/writer.h" +#include "arrow/record_batch.h" +#include "arrow/status.h" +#include "arrow/type.h" + +namespace arrow { +namespace compute { +namespace { + +constexpr uint8_t kFuzzHeaderMarker = 0xFA; + +Result> MakeNumericBatch() { + // int64 with nulls and negatives, float64 with fractions + Int64Builder i64_builder; + DoubleBuilder f64_builder; + const std::vector> ints = {0, -3, 1000, std::nullopt, 42, + -1000000, 7, 25}; + const std::vector> doubles = { + 0.0, -1.5, 3.14159, std::nullopt, 1e10, -0.001, 2.5, 4.0}; + for (const auto& v : ints) { + RETURN_NOT_OK(v.has_value() ? i64_builder.Append(*v) : i64_builder.AppendNull()); + } + for (const auto& v : doubles) { + RETURN_NOT_OK(v.has_value() ? f64_builder.Append(*v) : f64_builder.AppendNull()); + } + ARROW_ASSIGN_OR_RAISE(auto i64_array, i64_builder.Finish()); + ARROW_ASSIGN_OR_RAISE(auto f64_array, f64_builder.Finish()); + auto schema = ::arrow::schema({field("i", int64()), field("d", float64())}); + return RecordBatch::Make(schema, i64_array->length(), {i64_array, f64_array}); +} + +Result> MakeStringBatch() { + StringBuilder builder; + const std::vector> strings = { + std::string(""), std::string("hello"), std::nullopt, std::string("wörld"), + std::string("a longer string value for the kernels to chew on")}; + for (const auto& v : strings) { + RETURN_NOT_OK(v.has_value() ? builder.Append(*v) : builder.AppendNull()); + } + ARROW_ASSIGN_OR_RAISE(auto array, builder.Finish()); + auto schema = ::arrow::schema({field("s", utf8())}); + return RecordBatch::Make(schema, array->length(), {array}); +} + +Result> MakeSmallIntBatch() { + Int32Builder i32_builder; + const std::vector> ints = {1, std::nullopt, 3, 4}; + for (const auto& v : ints) { + RETURN_NOT_OK(v.has_value() ? i32_builder.Append(*v) : i32_builder.AppendNull()); + } + ARROW_ASSIGN_OR_RAISE(auto array, i32_builder.Finish()); + auto schema = ::arrow::schema({field("i", int32())}); + return RecordBatch::Make(schema, array->length(), {array}); +} + +Result> SerializeBatch(const RecordBatch& batch) { + ARROW_ASSIGN_OR_RAISE(auto sink, io::BufferOutputStream::Create(1024)); + ARROW_ASSIGN_OR_RAISE(auto writer, ipc::MakeStreamWriter(sink, batch.schema())); + RETURN_NOT_OK(writer->WriteRecordBatch(batch)); + RETURN_NOT_OK(writer->Close()); + return sink->Finish(); +} + +Status WriteSeed(const std::string& dir, int num, const uint8_t* header, + int64_t header_size, const std::shared_ptr& payload) { + auto name = dir + "/seed-" + std::to_string(num); + ARROW_ASSIGN_OR_RAISE(auto file, io::FileOutputStream::Open(name)); + if (header_size > 0) { + RETURN_NOT_OK(file->Write(header, header_size)); + } + RETURN_NOT_OK(file->Write(payload)); + RETURN_NOT_OK(file->Close()); + std::cerr << name << std::endl; + return Status::OK(); +} + +Status DoMain(const std::string& out_dir) { + // The oss-fuzz build script removes the corpus directory before running + // this generator; re-create it before writing seeds. + std::error_code ec; + std::filesystem::create_directories(out_dir, ec); + if (ec) { + return Status::IOError("Failed to create corpus directory '", out_dir, + "': ", ec.message()); + } + std::vector> batches; + ARROW_ASSIGN_OR_RAISE(auto numeric, MakeNumericBatch()); + batches.push_back(numeric); + ARROW_ASSIGN_OR_RAISE(auto strings, MakeStringBatch()); + batches.push_back(strings); + ARROW_ASSIGN_OR_RAISE(auto small, MakeSmallIntBatch()); + batches.push_back(small); + + int seed_num = 0; + // Legacy (header-less) seeds + for (const auto& batch : batches) { + ARROW_ASSIGN_OR_RAISE(auto payload, SerializeBatch(*batch)); + RETURN_NOT_OK(WriteSeed(out_dir, seed_num++, nullptr, 0, payload)); + } + // Header-ful seeds covering unary/vector/aggregate/binary functions + // (selector indices match FuzzFunctions() in fuzz.cc). + const uint8_t selectors[] = {0, 5, 8, 13, 16}; + for (const auto& batch : batches) { + ARROW_ASSIGN_OR_RAISE(auto payload, SerializeBatch(*batch)); + for (uint8_t selector : selectors) { + for (uint8_t arg_mode = 0; arg_mode < 3; ++arg_mode) { + uint8_t header[8] = {kFuzzHeaderMarker, selector, arg_mode, 7, + 0, 0, 0, 0}; + RETURN_NOT_OK(WriteSeed(out_dir, seed_num++, header, sizeof(header), payload)); + } + } + } + return Status::OK(); +} + +} // namespace + +Status GenerateFuzzCorpusMain(const std::string& out_dir) { return DoMain(out_dir); } + +} // namespace compute +} // namespace arrow + +int main(int argc, char** argv) { + if (argc != 2) { + std::cerr << "Usage: arrow-compute-generate-fuzz-corpus " + << std::endl; + return 2; + } + arrow::Status st = arrow::compute::GenerateFuzzCorpusMain(argv[1]); + if (!st.ok()) { + std::cerr << st.ToString() << std::endl; + return 1; + } + return 0; +} \ No newline at end of file