From 51d7ad80aa913d3f51065fd2bbbd0419056eb1fd Mon Sep 17 00:00:00 2001 From: "Brian D. Klahn" Date: Thu, 5 Nov 2026 22:15:36 -0500 Subject: [PATCH 1/2] Fix Utf8View/BinaryView variadicBufferCounts Int64 deserialization RecordBatch.variadicBufferCounts is defined as [long] (Int64) in the Apache Arrow Message.fbs spec, but was deserialized as FlatBuffers.Array{Int32}. When reading tables with multiple Utf8View/BinaryView columns (e.g. from Polars), reading 32-bit elements split each 64-bit buffer count into two 32-bit words, shifting buffer offsets for subsequent columns and causing out-of-bounds reads or segmentation faults in Arrow.View getindex: signal 11 (2): Segmentation fault: 11 getindex at ./essentials.jl:975 [inlined] getindex at Arrow/src/arraytypes/views.jl:61 [inlined] Also handle 4-byte-aligned Int64 vectors in FlatBuffers.Array{T} when the vector payload following the 4-byte UOffsetT length prefix is not 8-byte aligned. --- src/FlatBuffers/table.jl | 7 ++++++- src/metadata/Message.jl | 2 +- test/runtests.jl | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/FlatBuffers/table.jl b/src/FlatBuffers/table.jl index a17d1cd0..c0fef680 100644 --- a/src/FlatBuffers/table.jl +++ b/src/FlatBuffers/table.jl @@ -110,7 +110,12 @@ function Array{T}(t::Table, off) where {T} a = vector(t, off) S = T <: Table ? UOffsetT : T <: Struct ? NTuple{structsizeof(T),UInt8} : T ptr = convert(Ptr{S}, pointer(bytes(t), a + 1)) - data = unsafe_wrap(Base.Array, ptr, vectorlen(t, off)) + len = vectorlen(t, off) + data = if UInt(ptr) % Base.datatype_alignment(S) == 0 + unsafe_wrap(Base.Array, ptr, len) + else + copy(reinterpret(S, @view bytes(t)[(a + 1):(a + len * sizeof(S))])) + end return Array{T,S,typeof(t)}(t, a, data) end diff --git a/src/metadata/Message.jl b/src/metadata/Message.jl index 0e494394..126297f8 100644 --- a/src/metadata/Message.jl +++ b/src/metadata/Message.jl @@ -101,7 +101,7 @@ function Base.getproperty(x::RecordBatch, field::Symbol) elseif field === :variadicBufferCounts o = FlatBuffers.offset(x, 12) if o != 0 - return FlatBuffers.Array{Int32}(x, o) + return FlatBuffers.Array{Int64}(x, o) end end return nothing diff --git a/test/runtests.jl b/test/runtests.jl index 4f7d28b2..a19b4a3a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1107,6 +1107,22 @@ end joinpath(dirname(pathof(Arrow)), "../test/reject_reason_trimmed.arrow"), ) @test t.reject_reason[end] == "POST_ONLY" + + # RecordBatch.variadicBufferCounts uses 64-bit integers in the Arrow spec + FB = Arrow.FlatBuffers + b = FB.Builder(64) + FB.startvector!(b, sizeof(Int64), 3, sizeof(Int64)) + for cnt in Iterators.reverse(Int64[0, 7, 0]) + FB.prepend!(b, cnt) + end + varcounts = FB.endvector!(b, 3) + FB.startobject!(b, 5) + FB.prependoffsetslot!(b, 4, varcounts, 0) # slot 4 (offset 12) = variadicBufferCounts + FB.finish!(b, Arrow.Meta.recordBatchEnd(b)) + + rb = FB.getrootas(Arrow.Meta.RecordBatch, Vector(FB.finishedbytes(b)), 0) + @test eltype(rb.variadicBufferCounts) === Int64 + @test collect(rb.variadicBufferCounts) == Int64[0, 7, 0] end end # @testset "misc" From 94027fa979d7172d2a3bddcf470269a7adbbbcd7 Mon Sep 17 00:00:00 2001 From: "Brian D. Klahn" Date: Thu, 5 Nov 2026 22:44:18 -0500 Subject: [PATCH 2/2] Avoid copying unaligned FlatBuffers.Array vectors --- src/FlatBuffers/table.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/FlatBuffers/table.jl b/src/FlatBuffers/table.jl index c0fef680..a430ea50 100644 --- a/src/FlatBuffers/table.jl +++ b/src/FlatBuffers/table.jl @@ -100,10 +100,10 @@ function vector(t::Table, off) return x + sizeof(UOffsetT) end -struct Array{T,S,TT} <: AbstractVector{T} +struct Array{T,S,TT,D<:AbstractVector{S}} <: AbstractVector{T} _tab::TT pos::Int64 - data::Vector{S} + data::D end function Array{T}(t::Table, off) where {T} @@ -114,9 +114,9 @@ function Array{T}(t::Table, off) where {T} data = if UInt(ptr) % Base.datatype_alignment(S) == 0 unsafe_wrap(Base.Array, ptr, len) else - copy(reinterpret(S, @view bytes(t)[(a + 1):(a + len * sizeof(S))])) + reinterpret(S, @view bytes(t)[(a + 1):(a + len * sizeof(S))]) end - return Array{T,S,typeof(t)}(t, a, data) + return Array{T,S,typeof(t),typeof(data)}(t, a, data) end function structsizeof end