diff --git a/src/FlatBuffers/table.jl b/src/FlatBuffers/table.jl index a17d1cd0..b787b1ab 100644 --- a/src/FlatBuffers/table.jl +++ b/src/FlatBuffers/table.jl @@ -110,7 +110,17 @@ function Array{T}(t::Table, off) where {T} a = vector(t, off) S = T <: Table ? UOffsetT : T <: Struct ? NTuple{structsizeof(T),UInt8} : T ptr = convert(Ptr{S}, pointer(bytes(t), a + 1)) - data = unsafe_wrap(Base.Array, ptr, vectorlen(t, off)) + len = vectorlen(t, off) + if UInt(ptr) % Base.datatype_alignment(S) == 0 + data = unsafe_wrap(Base.Array, ptr, len) + else + # IPC metadata from other writers may not align scalar vectors for Julia. + nbytes = Base.checked_mul(Int(len), sizeof(S)) + first = Base.checked_add(a, 1) + last = Base.checked_add(a, nbytes) + checkbounds(bytes(t), first:last) + data = copy(reinterpret(S, @view bytes(t)[first:last])) + end return Array{T,S,typeof(t)}(t, a, data) end diff --git a/src/metadata/Message.jl b/src/metadata/Message.jl index 0e494394..126297f8 100644 --- a/src/metadata/Message.jl +++ b/src/metadata/Message.jl @@ -101,7 +101,7 @@ function Base.getproperty(x::RecordBatch, field::Symbol) elseif field === :variadicBufferCounts o = FlatBuffers.offset(x, 12) if o != 0 - return FlatBuffers.Array{Int32}(x, o) + return FlatBuffers.Array{Int64}(x, o) end end return nothing diff --git a/test/polars-variadic-buffer-counts.arrow b/test/polars-variadic-buffer-counts.arrow new file mode 100644 index 00000000..285c81af Binary files /dev/null and b/test/polars-variadic-buffer-counts.arrow differ diff --git a/test/runtests.jl b/test/runtests.jl index 315d1b60..5ef28cb5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1084,6 +1084,75 @@ end ) @test t.reject_reason[end] == "POST_ONLY" end + + @testset "#590: variadic buffer counts use 64-bit entries" begin + # PyArrow 25.0.1 fixture: text, binary, inline, nested text, then integers. + file = joinpath(@__DIR__, "variadic-buffer-counts.arrow") + batches = Arrow.BatchIterator(Arrow.ArrowBlob(read(file), 1, nothing)) + rb = only( + b.msg.header for b in batches if b.msg.header isa Arrow.Meta.RecordBatch + ) + @test eltype(rb.variadicBufferCounts) === Int64 + @test collect(rb.variadicBufferCounts) == [1, 2, 0, 1] + offset = Arrow.FlatBuffers.vector(rb, Arrow.FlatBuffers.offset(rb, 12)) + @test pointer(rb.variadicBufferCounts.data) == + pointer(Arrow.FlatBuffers.bytes(rb), offset + 1) + for t in (Arrow.Table(file), only(Arrow.Stream(file))) + @test isequal(t.text, ["नमस्ते", "long enough text", missing, ""]) + @test isequal( + t.binary, + [ + codeunits("first binary value"), + codeunits("second binary value"), + missing, + codeunits("x"), + ], + ) + @test isequal(t.inline, ["a", "b", missing, ""]) + @test isequal( + t.nested, + [ + (value="nested long value",), + (value=missing,), + (value="x",), + (value="another long value",), + ], + ) + @test t.numbers == Int32[1, 2, 3, 4] + end + + # Polars 1.21.0 stores these Int64 counts at a four-byte-aligned address. + file = joinpath(@__DIR__, "polars-variadic-buffer-counts.arrow") + bytes = read(file) + batches = Arrow.BatchIterator(Arrow.ArrowBlob(bytes, 1, nothing)) + rb = only( + b.msg.header for b in batches if b.msg.header isa Arrow.Meta.RecordBatch + ) + counts = rb.variadicBufferCounts + offset = Arrow.FlatBuffers.vector(rb, Arrow.FlatBuffers.offset(rb, 12)) + @test counts == [1, 1] + @test UInt(pointer(bytes, offset + 1)) % 8 == 4 + if Base.datatype_alignment(Int64) > 4 + @test pointer(counts.data) != pointer(bytes, offset + 1) + truncated = Arrow.Meta.RecordBatch( + bytes[1:(offset + 15)], + Arrow.FlatBuffers.pos(rb), + ) + @test_throws BoundsError truncated.variadicBufferCounts + oversized = copy(bytes) + fill!(@view(oversized[(offset - 3):offset]), 0xff) + @test_throws BoundsError Arrow.Meta.RecordBatch( + oversized, + Arrow.FlatBuffers.pos(rb), + ).variadicBufferCounts + else + @test pointer(counts.data) == pointer(bytes, offset + 1) + end + for t in (Arrow.Table(file), only(Arrow.Stream(file))) + @test t.more == ["नमस्ते"^2, "नमस्ते"^20] + @test t.text == ["k"^2, "k"^20] + end + end end # @testset "misc" @testset "DataAPI.metadata" begin diff --git a/test/variadic-buffer-counts.arrow b/test/variadic-buffer-counts.arrow new file mode 100644 index 00000000..340f9f67 Binary files /dev/null and b/test/variadic-buffer-counts.arrow differ