You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2022/11/03 11:08:47 UTC
[GitHub] [arrow-julia] ericphanson commented on pull request #357: Ensure Julia types have alignment respected
ericphanson commented on PR #357:
URL: https://github.com/apache/arrow-julia/pull/357#issuecomment-1301943392
On main, without rosetta on my m1 macbook, I get 4 failing tests:
```julia
Test Summary: | Pass Error Total Time
Arrow | 120 4 124 1m15.2s
table roundtrips | 44 1 45 17.4s
table append | 9 1 10 3.2s
arrow json integration tests | 1 1 2 29.4s
abstract path | 3 3 1.0s
misc | 63 1 64 24.1s
```
<details><summary>Stacktraces:</summary>
```julia
table roundtrips: Error During Test at /Users/eph/arrow-julia/test/runtests.jl:39
Got exception outside of a @test
TaskFailedException
nested task error: ArgumentError: unsafe_wrap: pointer 0x12bae16a8 is not properly aligned to 16 bytes
Stacktrace:
[1] #unsafe_wrap#102
@ ./pointer.jl:89 [inlined]
[2] unsafe_wrap
@ ./pointer.jl:87 [inlined]
[3] reinterp(#unused#::Type{Arrow.Decimal{2, 2, Int128}}, batch::Arrow.Batch, buf::Arrow.Flatbuf.Buffer, compression::Nothing)
@ Arrow ~/arrow-julia/src/table.jl:515
[4] build(f::Arrow.Flatbuf.Field, #unused#::Arrow.Flatbuf.Decimal, batch::Arrow.Batch, rb::Arrow.Flatbuf.RecordBatch, de::Dict{Int64, Arrow.DictEncoding}, nodeidx::Int64, bufferidx::Int64, convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:647
[5] build(field::Arrow.Flatbuf.Field, batch::Arrow.Batch, rb::Arrow.Flatbuf.RecordBatch, de::Dict{Int64, Arrow.DictEncoding}, nodeidx::Int64, bufferidx::Int64, convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:476
[6] iterate(x::Arrow.VectorIterator, ::Tuple{Int64, Int64, Int64})
@ Arrow ~/arrow-julia/src/table.jl:452
[7] iterate
@ ~/arrow-julia/src/table.jl:449 [inlined]
[8] copyto!(dest::Vector{Any}, src::Arrow.VectorIterator)
@ Base ./abstractarray.jl:901
[9] _collect
@ ./array.jl:718 [inlined]
[10] collect
@ ./array.jl:712 [inlined]
[11] macro expansion
@ ~/arrow-julia/src/table.jl:354 [inlined]
[12] (::Arrow.var"#108#114"{Bool, Channel{Any}, WorkerUtilities.OrderedSynchronizer, Dict{Int64, Arrow.DictEncoding}, Arrow.Batch, Int64})()
@ Arrow ./threadingconstructs.jl:258
Stacktrace:
[1] sync_end(c::Channel{Any})
@ Base ./task.jl:436
[2] macro expansion
@ ./task.jl:455 [inlined]
[3] Arrow.Table(blobs::Vector{Arrow.ArrowBlob}; convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:299
[4] Arrow.Table(input::IOBuffer, pos::Int64, len::Nothing; kw::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:convert,), Tuple{Bool}}})
@ Arrow ~/arrow-julia/src/table.jl:268
[5] testtable(nm::String, t::NamedTuple{(:col14, :col15, :col16, :col17, :col18, :col19, :col20), Tuple{Vector{Union{Missing, Arrow.Decimal{2, 2, Int128}}}, Vector{Union{Missing, Arrow.Date{Arrow.Flatbuf.DateUnits.DAY, Int32}}}, Vector{Union{Missing, Arrow.Time{Arrow.Flatbuf.TimeUnits.SECOND, Int32}}}, Vector{Union{Missing, Arrow.Timestamp{Arrow.Flatbuf.TimeUnits.SECOND, nothing}}}, Vector{Union{Missing, Arrow.Interval{Arrow.Flatbuf.IntervalUnits.YEAR_MONTH, Int32}}}, Vector{Union{Missing, Arrow.Duration{Arrow.Flatbuf.TimeUnits.SECOND}}}, Vector{Union{Missing, Arrow.Date{Arrow.Flatbuf.DateUnits.MILLISECOND, Int64}}}}}, writekw::NamedTuple{(), Tuple{}}, readkw::NamedTuple{(:convert,), Tuple{Bool}}, extratests::Nothing)
@ Main ~/arrow-julia/test/testtables.jl:247
[6] macro expansion
@ ~/arrow-julia/test/runtests.jl:42 [inlined]
[7] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[8] macro expansion
@ ~/arrow-julia/test/runtests.jl:41 [inlined]
[9] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[10] top-level scope
@ ~/arrow-julia/test/runtests.jl:39
[11] include(fname::String)
@ Base.MainInclude ./client.jl:476
[12] top-level scope
@ none:6
[13] eval
@ ./boot.jl:368 [inlined]
[14] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:276
[15] _start()
@ Base ./client.jl:522
testing append: basic
testing append: missing values
testing append: primitive types
testing append: arrow date/time types
table append: Error During Test at /Users/eph/arrow-julia/test/runtests.jl:47
Got exception outside of a @test
ArgumentError: unsafe_wrap: pointer 0x1760fc368 is not properly aligned to 16 bytes
Stacktrace:
[1] #unsafe_wrap#102
@ ./pointer.jl:89 [inlined]
[2] unsafe_wrap
@ ./pointer.jl:87 [inlined]
[3] reinterp(#unused#::Type{Arrow.Decimal{2, 2, Int128}}, batch::Arrow.Batch, buf::Arrow.Flatbuf.Buffer, compression::Nothing)
@ Arrow ~/arrow-julia/src/table.jl:515
[4] build(f::Arrow.Flatbuf.Field, #unused#::Arrow.Flatbuf.Decimal, batch::Arrow.Batch, rb::Arrow.Flatbuf.RecordBatch, de::Dict{Int64, Arrow.DictEncoding}, nodeidx::Int64, bufferidx::Int64, convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:647
[5] build(field::Arrow.Flatbuf.Field, batch::Arrow.Batch, rb::Arrow.Flatbuf.RecordBatch, de::Dict{Int64, Arrow.DictEncoding}, nodeidx::Int64, bufferidx::Int64, convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:476
[6] iterate(x::Arrow.VectorIterator, ::Tuple{Int64, Int64, Int64})
@ Arrow ~/arrow-julia/src/table.jl:452
[7] iterate
@ ~/arrow-julia/src/table.jl:449 [inlined]
[8] iterate(x::Arrow.Stream, ::Tuple{Int64, Int64})
@ Arrow ~/arrow-julia/src/table.jl:169
[9] iterate
@ ~/arrow-julia/src/table.jl:102 [inlined]
[10] stream_properties(io::IOStream; convert::Bool)
@ Arrow ~/arrow-julia/src/append.jl:170
[11] append(io::IOStream, tbl::Arrow.Table; metadata::Nothing, colmetadata::Nothing, largelists::Bool, denseunions::Bool, dictencode::Bool, dictencodenested::Bool, alignment::Int64, maxdepth::Int64, ntasks::Float64, convert::Bool, file::Bool)
@ Arrow ~/arrow-julia/src/append.jl:87
[12] #153
@ ~/arrow-julia/src/append.jl:64 [inlined]
[13] open(::Arrow.var"#153#154"{Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:convert,), Tuple{Bool}}}, Arrow.Table}, ::String, ::Vararg{String}; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ Base ./io.jl:384
[14] open
@ ./io.jl:381 [inlined]
[15] #append#152
@ ~/arrow-julia/src/append.jl:63 [inlined]
[16] (::var"#5#6"{NamedTuple{(), Tuple{}}, NamedTuple{(:convert,), Tuple{Bool}}, Vector{UInt8}})(path::String, io::IOStream)
@ Main ~/arrow-julia/test/testappend.jl:28
[17] mktemp(fn::var"#5#6"{NamedTuple{(), Tuple{}}, NamedTuple{(:convert,), Tuple{Bool}}, Vector{UInt8}}, parent::String)
@ Base.Filesystem ./file.jl:736
[18] mktemp
@ ./file.jl:734 [inlined]
[19] testappend(nm::String, t::NamedTuple{(:col14, :col15, :col16, :col17, :col18, :col19, :col20), Tuple{Vector{Union{Missing, Arrow.Decimal{2, 2, Int128}}}, Vector{Union{Missing, Arrow.Date{Arrow.Flatbuf.DateUnits.DAY, Int32}}}, Vector{Union{Missing, Arrow.Time{Arrow.Flatbuf.TimeUnits.SECOND, Int32}}}, Vector{Union{Missing, Arrow.Timestamp{Arrow.Flatbuf.TimeUnits.SECOND, nothing}}}, Vector{Union{Missing, Arrow.Interval{Arrow.Flatbuf.IntervalUnits.YEAR_MONTH, Int32}}}, Vector{Union{Missing, Arrow.Duration{Arrow.Flatbuf.TimeUnits.SECOND}}}, Vector{Union{Missing, Arrow.Date{Arrow.Flatbuf.DateUnits.MILLISECOND, Int64}}}}}, writekw::NamedTuple{(), Tuple{}}, readkw::NamedTuple{(:convert,), Tuple{Bool}}, extratests::Nothing)
@ Main ~/arrow-julia/test/testappend.jl:22
[20] macro expansion
@ ~/arrow-julia/test/runtests.jl:50 [inlined]
[21] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[22] macro expansion
@ ~/arrow-julia/test/runtests.jl:49 [inlined]
[23] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[24] top-level scope
@ ~/arrow-julia/test/runtests.jl:39
[25] include(fname::String)
@ Base.MainInclude ./client.jl:476
[26] top-level scope
@ none:6
[27] eval
@ ./boot.jl:368 [inlined]
[28] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:276
[29] _start()
@ Base ./client.jl:522
integration test for /Users/eph/arrow-julia/src/../test/arrowjson/datetime.json
integration test for /Users/eph/arrow-julia/src/../test/arrowjson/decimal.json
arrow json integration tests: Error During Test at /Users/eph/arrow-julia/test/runtests.jl:61
Got exception outside of a @test
TaskFailedException
nested task error: ArgumentError: unsafe_wrap: pointer 0x178c18ef8 is not properly aligned to 16 bytes
Stacktrace:
[1] #unsafe_wrap#102
@ ./pointer.jl:89 [inlined]
[2] unsafe_wrap
@ ./pointer.jl:87 [inlined]
[3] reinterp(#unused#::Type{Arrow.Decimal{4, 2, Int128}}, batch::Arrow.Batch, buf::Arrow.Flatbuf.Buffer, compression::Nothing)
@ Arrow ~/arrow-julia/src/table.jl:515
[4] build(f::Arrow.Flatbuf.Field, #unused#::Arrow.Flatbuf.Decimal, batch::Arrow.Batch, rb::Arrow.Flatbuf.RecordBatch, de::Dict{Int64, Arrow.DictEncoding}, nodeidx::Int64, bufferidx::Int64, convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:647
[5] build(field::Arrow.Flatbuf.Field, batch::Arrow.Batch, rb::Arrow.Flatbuf.RecordBatch, de::Dict{Int64, Arrow.DictEncoding}, nodeidx::Int64, bufferidx::Int64, convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:476
[6] iterate(x::Arrow.VectorIterator, ::Tuple{Int64, Int64, Int64})
@ Arrow ~/arrow-julia/src/table.jl:452
[7] copyto!(dest::Vector{Any}, src::Arrow.VectorIterator)
@ Base ./abstractarray.jl:906
[8] _collect
@ ./array.jl:718 [inlined]
[9] collect
@ ./array.jl:712 [inlined]
[10] macro expansion
@ ~/arrow-julia/src/table.jl:354 [inlined]
[11] (::Arrow.var"#108#114"{Bool, Channel{Any}, WorkerUtilities.OrderedSynchronizer, Dict{Int64, Arrow.DictEncoding}, Arrow.Batch, Int64})()
@ Arrow ./threadingconstructs.jl:258
...and 35 more exceptions.
Stacktrace:
[1] sync_end(c::Channel{Any})
@ Base ./task.jl:436
[2] macro expansion
@ ./task.jl:455 [inlined]
[3] Arrow.Table(blobs::Vector{Arrow.ArrowBlob}; convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:299
[4] Arrow.Table(input::IOBuffer, pos::Int64, len::Nothing; kw::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:convert,), Tuple{Bool}}})
@ Arrow ~/arrow-julia/src/table.jl:268
[5] macro expansion
@ ~/arrow-julia/test/runtests.jl:68 [inlined]
[6] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[7] macro expansion
@ ~/arrow-julia/test/runtests.jl:63 [inlined]
[8] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[9] top-level scope
@ ~/arrow-julia/test/runtests.jl:39
[10] include(fname::String)
@ Base.MainInclude ./client.jl:476
[11] top-level scope
@ none:6
[12] eval
@ ./boot.jl:368 [inlined]
[13] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:276
[14] _start()
@ Base ./client.jl:522
misc: Error During Test at /Users/eph/arrow-julia/test/runtests.jl:98
Got exception outside of a @test
TaskFailedException
Stacktrace:
[1] wait
@ ./task.jl:345 [inlined]
[2] Arrow.Table(blobs::Vector{Arrow.ArrowBlob}; convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:364
[3] Table
@ ~/arrow-julia/src/table.jl:273 [inlined]
[4] Arrow.Table(input::IOBuffer, pos::Int64, len::Nothing; kw::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ Arrow ~/arrow-julia/src/table.jl:268
[5] Table (repeats 2 times)
@ ~/arrow-julia/src/table.jl:268 [inlined]
[6] macro expansion
@ ~/arrow-julia/test/runtests.jl:294 [inlined]
[7] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[8] macro expansion
@ ~/arrow-julia/test/runtests.jl:101 [inlined]
[9] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[10] top-level scope
@ ~/arrow-julia/test/runtests.jl:39
[11] include(fname::String)
@ Base.MainInclude ./client.jl:476
[12] top-level scope
@ none:6
[13] eval
@ ./boot.jl:368 [inlined]
[14] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:276
[15] _start()
@ Base ./client.jl:522
nested task error: MethodError: no method matching resize!(::Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}, ::Int64)
Closest candidates are:
resize!(::Vector, ::Integer) at array.jl:1233
resize!(::BitVector, ::Integer) at bitarray.jl:814
resize!(::SentinelArrays.ChainedVector{T, AT}, ::Any) where {T, AT} at ~/.julia/packages/SentinelArrays/XvFr7/src/chainedvector.jl:481
...
Stacktrace:
[1] _append!(a::Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}, #unused#::Base.HasLength, iter::Tuple{Int64})
@ Base ./array.jl:1121
[2] append!(a::Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}, iter::Tuple{Int64})
@ Base ./array.jl:1113
[3] push!(a::Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}, iter::Int64)
@ Base ./array.jl:1114
[4] push!(A::SentinelArrays.ChainedVector{Int64, Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}}, val::Int64)
@ SentinelArrays ~/.julia/packages/SentinelArrays/XvFr7/src/chainedvector.jl:506
[5] append!(A::SentinelArrays.ChainedVector{Int64, Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}}, B::Arrow.DictEncoded{Int64, Int8, SentinelArrays.ChainedVector{Int64, Arrow.Primitive{Int64, Vector{Int64}}}})
@ SentinelArrays ~/.julia/packages/SentinelArrays/XvFr7/src/chainedvector.jl:664
[6] (::Arrow.var"#107#113"{Vector{Any}, Arrow.Table})(i::Int64)
@ Arrow ~/arrow-julia/src/table.jl:291
[7] foreach(f::Arrow.var"#107#113"{Vector{Any}, Arrow.Table}, itr::UnitRange{Int64})
@ Base ./abstractarray.jl:2774
[8] macro expansion
@ ~/arrow-julia/src/table.jl:0 [inlined]
[9] (::Arrow.var"#104#110"{Channel{Any}, Arrow.Table})()
@ Arrow ./threadingconstructs.jl:258
```
</details>
On this PR, I get 1 failing test still:
```julia
Test Summary: | Pass Error Total Time
Arrow | 381 1 382 2m16.7s
table roundtrips | 219 219 56.9s
table append | 87 87 14.6s
arrow json integration tests | 9 9 50.2s
abstract path | 3 3 0.8s
misc | 63 1 64 14.2s
```
<details><summary>Stacktrace:</summary>
```julia
misc: Error During Test at /Users/eph/arrow-julia/test/runtests.jl:98
Got exception outside of a @test
TaskFailedException
Stacktrace:
[1] wait
@ ./task.jl:345 [inlined]
[2] Arrow.Table(blobs::Vector{Arrow.ArrowBlob}; convert::Bool)
@ Arrow ~/arrow-julia/src/table.jl:364
[3] Table
@ ~/arrow-julia/src/table.jl:273 [inlined]
[4] Arrow.Table(input::IOBuffer, pos::Int64, len::Nothing; kw::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ Arrow ~/arrow-julia/src/table.jl:268
[5] Table (repeats 2 times)
@ ~/arrow-julia/src/table.jl:268 [inlined]
[6] macro expansion
@ ~/arrow-julia/test/runtests.jl:294 [inlined]
[7] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[8] macro expansion
@ ~/arrow-julia/test/runtests.jl:101 [inlined]
[9] macro expansion
@ ~/.julia/juliaup/julia-1.8.2+0.aarch64/share/julia/stdlib/v1.8/Test/src/Test.jl:1357 [inlined]
[10] top-level scope
@ ~/arrow-julia/test/runtests.jl:39
[11] include(fname::String)
@ Base.MainInclude ./client.jl:476
[12] top-level scope
@ none:6
[13] eval
@ ./boot.jl:368 [inlined]
[14] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:276
[15] _start()
@ Base ./client.jl:522
nested task error: MethodError: no method matching resize!(::Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}, ::Int64)
Closest candidates are:
resize!(::Vector, ::Integer) at array.jl:1233
resize!(::BitVector, ::Integer) at bitarray.jl:814
resize!(::SentinelArrays.ChainedVector{T, AT}, ::Any) where {T, AT} at ~/.julia/packages/SentinelArrays/XvFr7/src/chainedvector.jl:481
...
Stacktrace:
[1] _append!(a::Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}, #unused#::Base.HasLength, iter::Tuple{Int64})
@ Base ./array.jl:1121
[2] append!(a::Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}, iter::Tuple{Int64})
@ Base ./array.jl:1113
[3] push!(a::Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}, iter::Int64)
@ Base ./array.jl:1114
[4] push!(A::SentinelArrays.ChainedVector{Int64, Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}}, val::Int64)
@ SentinelArrays ~/.julia/packages/SentinelArrays/XvFr7/src/chainedvector.jl:506
[5] append!(A::SentinelArrays.ChainedVector{Int64, Arrow.DictEncoded{Int64, Int8, Arrow.Primitive{Int64, Vector{Int64}}}}, B::Arrow.DictEncoded{Int64, Int8, SentinelArrays.ChainedVector{Int64, Arrow.Primitive{Int64, Vector{Int64}}}})
@ SentinelArrays ~/.julia/packages/SentinelArrays/XvFr7/src/chainedvector.jl:664
[6] (::Arrow.var"#107#113"{Vector{Any}, Arrow.Table})(i::Int64)
@ Arrow ~/arrow-julia/src/table.jl:291
[7] foreach(f::Arrow.var"#107#113"{Vector{Any}, Arrow.Table}, itr::UnitRange{Int64})
@ Base ./abstractarray.jl:2774
[8] macro expansion
@ ~/arrow-julia/src/table.jl:0 [inlined]
[9] (::Arrow.var"#104#110"{Channel{Any}, Arrow.Table})()
@ Arrow ./threadingconstructs.jl:258
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org