diff --git a/src/table.jl b/src/table.jl index 606f2abe..fa5ec001 100644 --- a/src/table.jl +++ b/src/table.jl @@ -810,6 +810,7 @@ function reinterp(::Type{T}, batch, buf, compression) where {T} len = buf.length if compression !== nothing len, bytes = uncompress(ptr, buf, compression) + T === UInt8 && return bytes, bytes ptr = pointer(bytes) end # it would be technically more correct to check that T.layout->alignment > 8 @@ -897,7 +898,7 @@ function build( validity = buildbitmap(batch, rb, nodeidx, bufferidx) bufferidx += 1 buffer = rb.buffers[bufferidx] - _, views = reinterp(ViewElement, batch, buffer, rb.compression) + bytes, views = reinterp(ViewElement, batch, buffer, rb.compression) inline = reinterpret(UInt8, views) # reuse the (possibly realigned) memory backing `views` bufferidx += 1 buffers = Vector{UInt8}[] @@ -912,7 +913,7 @@ function build( nodeidx += 1 meta = buildmetadata(f.custom_metadata) T = juliaeltype(f, meta, convert) - return View{T}(batch.bytes, validity, views, inline, buffers, len, meta), + return View{T}(bytes, validity, views, inline, buffers, len, meta), nodeidx, bufferidx, varbufferidx diff --git a/test/runtests.jl b/test/runtests.jl index a19b4a3a..78cdc099 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1103,9 +1103,8 @@ end end @testset "#511: Bug in reading Utf8View data" begin - t = Arrow.Table( - joinpath(dirname(pathof(Arrow)), "../test/reject_reason_trimmed.arrow"), - ) + p = joinpath(dirname(pathof(Arrow)), "../test/reject_reason_trimmed.arrow") + t = Arrow.Table(p) @test t.reject_reason[end] == "POST_ONLY" # RecordBatch.variadicBufferCounts uses 64-bit integers in the Arrow spec @@ -1123,6 +1122,53 @@ end rb = FB.getrootas(Arrow.Meta.RecordBatch, Vector(FB.finishedbytes(b)), 0) @test eltype(rb.variadicBufferCounts) === Int64 @test collect(rb.variadicBufferCounts) == Int64[0, 7, 0] + + # Compressed Utf8View buffers stay GC-rooted in reinterp and Arrow.View + zstd_frame(b::Vector{UInt8}) = vcat( + reinterpret(UInt8, [Int64(length(b))]), + transcode(Arrow.CodecZstd.ZstdCompressor, b), + ) + raw = collect(codeunits("Luxembourg_120203202_18815")) + views_raw = collect( + reinterpret( + UInt8, + [ + Arrow.ViewElement( + Int32(5), + Int32(0x726f6873), + Int32(116), + Int32(0), + ), + Arrow.ViewElement( + Int32(length(raw)), + reinterpret(Int32, raw[1:4])[1], + Int32(0), + Int32(0), + ), + ], + ), + ) + c_views = zstd_frame(views_raw) + c_buf = zstd_frame(raw) + batch = (; bytes=vcat(c_views, c_buf), pos=1) + rb = (; + buffers=[ + (; offset=0, length=0), + (; offset=0, length=length(c_views)), + (; offset=length(c_views), length=length(c_buf)), + ], + nodes=[(; length=2, null_count=0)], + compression=(; codec=Arrow.Meta.CompressionType.ZSTD), + variadicBufferCounts=[1], + ) + bytes, A = Arrow.reinterp(UInt8, batch, rb.buffers[3], rb.compression) + @test bytes === A + f = + first(Arrow.BatchIterator(Arrow.ArrowBlob(Arrow.tobytes(p), 1, nothing))).msg.header.fields[1] + col, _ = Arrow.build(f, f.type, batch, rb, nothing, 1, 1, 1, true) + @test col.arrow !== batch.bytes + GC.gc() + @test collect(col) == ["short", "Luxembourg_120203202_18815"] end end # @testset "misc"