GH-590: Correct view buffer counts and handle unaligned metadata

src/FlatBuffers/table.jl

@@ -110,7 +110,17 @@ function Array{T}(t::Table, off) where {T}
     a = vector(t, off)
     S = T <: Table ? UOffsetT : T <: Struct ? NTuple{structsizeof(T),UInt8} : T
     ptr = convert(Ptr{S}, pointer(bytes(t), a + 1))
-    data = unsafe_wrap(Base.Array, ptr, vectorlen(t, off))
+    len = vectorlen(t, off)
+    if UInt(ptr) % Base.datatype_alignment(S) == 0
+        data = unsafe_wrap(Base.Array, ptr, len)
+    else
+        # IPC metadata from other writers may not align scalar vectors for Julia.
+        nbytes = Base.checked_mul(Int(len), sizeof(S))
+        first = Base.checked_add(a, 1)
+        last = Base.checked_add(a, nbytes)
+        checkbounds(bytes(t), first:last)
+        data = copy(reinterpret(S, @view bytes(t)[first:last]))
+    end
     return Array{T,S,typeof(t)}(t, a, data)
 end
 

src/metadata/Message.jl

@@ -101,7 +101,7 @@ function Base.getproperty(x::RecordBatch, field::Symbol)
     elseif field === :variadicBufferCounts
         o = FlatBuffers.offset(x, 12)
         if o != 0
-            return FlatBuffers.Array{Int32}(x, o)
+            return FlatBuffers.Array{Int64}(x, o)
         end
     end
     return nothing

test/runtests.jl

@@ -1084,6 +1084,75 @@ end
             )
             @test t.reject_reason[end] == "POST_ONLY"
         end
+
+        @testset "#590: variadic buffer counts use 64-bit entries" begin
+            # PyArrow 25.0.1 fixture: text, binary, inline, nested text, then integers.
+            file = joinpath(@__DIR__, "variadic-buffer-counts.arrow")
+            batches = Arrow.BatchIterator(Arrow.ArrowBlob(read(file), 1, nothing))
+            rb = only(
+                b.msg.header for b in batches if b.msg.header isa Arrow.Meta.RecordBatch
+            )
+            @test eltype(rb.variadicBufferCounts) === Int64
+            @test collect(rb.variadicBufferCounts) == [1, 2, 0, 1]
+            offset = Arrow.FlatBuffers.vector(rb, Arrow.FlatBuffers.offset(rb, 12))
+            @test pointer(rb.variadicBufferCounts.data) ==
+                  pointer(Arrow.FlatBuffers.bytes(rb), offset + 1)
+            for t in (Arrow.Table(file), only(Arrow.Stream(file)))
+                @test isequal(t.text, ["नमस्ते", "long enough text", missing, ""])
+                @test isequal(
+                    t.binary,
+                    [
+                        codeunits("first binary value"),
+                        codeunits("second binary value"),
+                        missing,
+                        codeunits("x"),
+                    ],
+                )
+                @test isequal(t.inline, ["a", "b", missing, ""])
+                @test isequal(
+                    t.nested,
+                    [
+                        (value="nested long value",),
+                        (value=missing,),
+                        (value="x",),
+                        (value="another long value",),
+                    ],
+                )
+                @test t.numbers == Int32[1, 2, 3, 4]
+            end
+
+            # Polars 1.21.0 stores these Int64 counts at a four-byte-aligned address.
+            file = joinpath(@__DIR__, "polars-variadic-buffer-counts.arrow")
+            bytes = read(file)
+            batches = Arrow.BatchIterator(Arrow.ArrowBlob(bytes, 1, nothing))
+            rb = only(
+                b.msg.header for b in batches if b.msg.header isa Arrow.Meta.RecordBatch
+            )
+            counts = rb.variadicBufferCounts
+            offset = Arrow.FlatBuffers.vector(rb, Arrow.FlatBuffers.offset(rb, 12))
+            @test counts == [1, 1]
+            @test UInt(pointer(bytes, offset + 1)) % 8 == 4
+            if Base.datatype_alignment(Int64) > 4
+                @test pointer(counts.data) != pointer(bytes, offset + 1)
+                truncated = Arrow.Meta.RecordBatch(
+                    bytes[1:(offset + 15)],
+                    Arrow.FlatBuffers.pos(rb),
+                )
+                @test_throws BoundsError truncated.variadicBufferCounts
+                oversized = copy(bytes)
+                fill!(@view(oversized[(offset - 3):offset]), 0xff)
+                @test_throws BoundsError Arrow.Meta.RecordBatch(
+                    oversized,
+                    Arrow.FlatBuffers.pos(rb),
+                ).variadicBufferCounts
+            else
+                @test pointer(counts.data) == pointer(bytes, offset + 1)
+            end
+            for t in (Arrow.Table(file), only(Arrow.Stream(file)))
+                @test t.more == ["नमस्ते"^2, "नमस्ते"^20]
+                @test t.text == ["k"^2, "k"^20]
+            end
+        end
     end # @testset "misc"
 
     @testset "DataAPI.metadata" begin