From df26dc51ab3366afb3413d980c10031d2ba1a216 Mon Sep 17 00:00:00 2001
From: Jacob Quinn <quinn.jacobd@gmail.com>
Date: Tue, 22 Sep 2026 08:13:51 -0600
Subject: [PATCH] GH-590: Correct view buffer counts and handle unaligned
 metadata

---
 src/FlatBuffers/table.jl                 |  12 +++-
 src/metadata/Message.jl                  |   2 +-
 test/polars-variadic-buffer-counts.arrow | Bin 0 -> 1327 bytes
 test/runtests.jl                         |  69 +++++++++++++++++++++++
 test/variadic-buffer-counts.arrow        | Bin 0 -> 1690 bytes
 5 files changed, 81 insertions(+), 2 deletions(-)
 create mode 100644 test/polars-variadic-buffer-counts.arrow
 create mode 100644 test/variadic-buffer-counts.arrow

diff --git a/src/FlatBuffers/table.jl b/src/FlatBuffers/table.jl
index a17d1cd0..b787b1ab 100644
--- a/src/FlatBuffers/table.jl
+++ b/src/FlatBuffers/table.jl
@@ -110,7 +110,17 @@ function Array{T}(t::Table, off) where {T}
     a = vector(t, off)
     S = T <: Table ? UOffsetT : T <: Struct ? NTuple{structsizeof(T),UInt8} : T
     ptr = convert(Ptr{S}, pointer(bytes(t), a + 1))
-    data = unsafe_wrap(Base.Array, ptr, vectorlen(t, off))
+    len = vectorlen(t, off)
+    if UInt(ptr) % Base.datatype_alignment(S) == 0
+        data = unsafe_wrap(Base.Array, ptr, len)
+    else
+        # IPC metadata from other writers may not align scalar vectors for Julia.
+        nbytes = Base.checked_mul(Int(len), sizeof(S))
+        first = Base.checked_add(a, 1)
+        last = Base.checked_add(a, nbytes)
+        checkbounds(bytes(t), first:last)
+        data = copy(reinterpret(S, @view bytes(t)[first:last]))
+    end
     return Array{T,S,typeof(t)}(t, a, data)
 end
 
diff --git a/src/metadata/Message.jl b/src/metadata/Message.jl
index 0e494394..126297f8 100644
--- a/src/metadata/Message.jl
+++ b/src/metadata/Message.jl
@@ -101,7 +101,7 @@ function Base.getproperty(x::RecordBatch, field::Symbol)
     elseif field === :variadicBufferCounts
         o = FlatBuffers.offset(x, 12)
         if o != 0
-            return FlatBuffers.Array{Int32}(x, o)
+            return FlatBuffers.Array{Int64}(x, o)
         end
     end
     return nothing
diff --git a/test/polars-variadic-buffer-counts.arrow b/test/polars-variadic-buffer-counts.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..285c81af599e0d8560fbc9babe2275a8f2d920d7
GIT binary patch
literal 1327
zcmeHHJxc>Y5Pi`_f*x|l6e+K;v=J;UFJLdE@fQSyLxhmR(#}TzgT)4{#M06~A}rQc
zRw;z|-rO!p#3W#&g%jpu-@MtKyWDKI*E=~q049`20FL_3xNUKRb`x7@(mITZXGEI(
ziMd95Y!UFr+&<%i+MzaRa<s_Gv15Ey0w3f#qB>{TzZ?qtN$vT?ps!kPHyP7wKkaj+
zrYLp)>&UQcePm?3G9S~pBj=E+UTfKA?8wi#Y}>Qx=_Sc4-jG=!Ma0YKZ&G18G~%{7
z<07uhnVx4U3(6A3{2p!a_aT92JTtA8TP5Q9hv1*W=r(vCM%VwyZ@rCG|J8SSeIr#h
z&FH(YC{tv4B<@#PUNN85{0Y&GAkhq#^heZs2EmzXajG0Sna$EOb0%><(#$6K&)qXG
J?5t}S<tH<@hkpP7

literal 0
HcmV?d00001

diff --git a/test/runtests.jl b/test/runtests.jl
index 315d1b60..5ef28cb5 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1084,6 +1084,75 @@ end
             )
             @test t.reject_reason[end] == "POST_ONLY"
         end
+
+        @testset "#590: variadic buffer counts use 64-bit entries" begin
+            # PyArrow 25.0.1 fixture: text, binary, inline, nested text, then integers.
+            file = joinpath(@__DIR__, "variadic-buffer-counts.arrow")
+            batches = Arrow.BatchIterator(Arrow.ArrowBlob(read(file), 1, nothing))
+            rb = only(
+                b.msg.header for b in batches if b.msg.header isa Arrow.Meta.RecordBatch
+            )
+            @test eltype(rb.variadicBufferCounts) === Int64
+            @test collect(rb.variadicBufferCounts) == [1, 2, 0, 1]
+            offset = Arrow.FlatBuffers.vector(rb, Arrow.FlatBuffers.offset(rb, 12))
+            @test pointer(rb.variadicBufferCounts.data) ==
+                  pointer(Arrow.FlatBuffers.bytes(rb), offset + 1)
+            for t in (Arrow.Table(file), only(Arrow.Stream(file)))
+                @test isequal(t.text, ["नमस्ते", "long enough text", missing, ""])
+                @test isequal(
+                    t.binary,
+                    [
+                        codeunits("first binary value"),
+                        codeunits("second binary value"),
+                        missing,
+                        codeunits("x"),
+                    ],
+                )
+                @test isequal(t.inline, ["a", "b", missing, ""])
+                @test isequal(
+                    t.nested,
+                    [
+                        (value="nested long value",),
+                        (value=missing,),
+                        (value="x",),
+                        (value="another long value",),
+                    ],
+                )
+                @test t.numbers == Int32[1, 2, 3, 4]
+            end
+
+            # Polars 1.21.0 stores these Int64 counts at a four-byte-aligned address.
+            file = joinpath(@__DIR__, "polars-variadic-buffer-counts.arrow")
+            bytes = read(file)
+            batches = Arrow.BatchIterator(Arrow.ArrowBlob(bytes, 1, nothing))
+            rb = only(
+                b.msg.header for b in batches if b.msg.header isa Arrow.Meta.RecordBatch
+            )
+            counts = rb.variadicBufferCounts
+            offset = Arrow.FlatBuffers.vector(rb, Arrow.FlatBuffers.offset(rb, 12))
+            @test counts == [1, 1]
+            @test UInt(pointer(bytes, offset + 1)) % 8 == 4
+            if Base.datatype_alignment(Int64) > 4
+                @test pointer(counts.data) != pointer(bytes, offset + 1)
+                truncated = Arrow.Meta.RecordBatch(
+                    bytes[1:(offset + 15)],
+                    Arrow.FlatBuffers.pos(rb),
+                )
+                @test_throws BoundsError truncated.variadicBufferCounts
+                oversized = copy(bytes)
+                fill!(@view(oversized[(offset - 3):offset]), 0xff)
+                @test_throws BoundsError Arrow.Meta.RecordBatch(
+                    oversized,
+                    Arrow.FlatBuffers.pos(rb),
+                ).variadicBufferCounts
+            else
+                @test pointer(counts.data) == pointer(bytes, offset + 1)
+            end
+            for t in (Arrow.Table(file), only(Arrow.Stream(file)))
+                @test t.more == ["नमस्ते"^2, "नमस्ते"^20]
+                @test t.text == ["k"^2, "k"^20]
+            end
+        end
     end # @testset "misc"
 
     @testset "DataAPI.metadata" begin
diff --git a/test/variadic-buffer-counts.arrow b/test/variadic-buffer-counts.arrow
new file mode 100644
index 0000000000000000000000000000000000000000..340f9f671fa588c6f0df1d3c03b26d27d757319f
GIT binary patch
literal 1690
zcmeHIJ#Q015FI;*ZNViNf*=LO2`DI`OOfKz(GjJgp}@vVV#%x{e<1S<l7^l##YIYm
zQsxIxP*SG6l9GQA-kX~N%fwP7T1J|EJ74c+=e+0b-QAb3o{L;U-qb|eLb)laG~|Y~
zu!GbR*aEACHL+C?XH4WP;4?IbCeSC~LZD8r%zOZ8LgVy(H;l&MNTE&Tg1eSsd0o}5
z-oySOlw6<}+6`z7<4Nd=>>%%`%H7~IHlWO<u%{25!88D8$T?e*N4|a>^dl0Fk#kYy
zJy_&pUbi1RqmLrrkkdvtEp*e=iL`BAHSNQDMv3+_3A0JHPk#3YipvWeif5vva^EJM
zDx<6<diFCOTxAV*8}*!hDr;Cb5!(W?S3ftly7#g=9dqJ-x^7?LCp#tARpf=?Bf|qj
z=b4$E8=f29HT-V4ZurB{>wlV^8$R>Eg6(denUd>@cWd@zvyVM-!RCxy_%3~3o_<~1
zIhN&n{FZGvhq5>=-OoM-!?;J=XXLS~khR5^vN$hKK9xmLo*c3|3voEz+t2)$B{KiK
zcYRze6V81c-VXho?suf$-ywahon+N7J@X3$KRIIW?|?s-JRK64(C)f%4P3RHJ3!v=
zWhIOKa)zBa)O@cEy-bIs?K8L^MspseCQlu@0_D5peFW%TN^vV2xMwLWe(#vEiS?=O
ds&CN&2_U86pL<&Q|9>O@?HhT7xo+!!(k~~rz|{Z%

literal 0
HcmV?d00001