You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@arrow.apache.org by GitBox <gi...@apache.org> on 2022/08/24 11:36:49 UTC
[GitHub] [arrow-julia] ericphanson opened a new issue, #332: Difficulties trying to serialize to Union types
ericphanson opened a new issue, #332:
URL: https://github.com/apache/arrow-julia/issues/332
Setup:
```julia
using Arrow
struct A0 end
struct A1
x::Int
end
struct A2
x::Int
y::Float64
end
ArrowTypes.arrowname(::Type{A0}) = :A0
ArrowTypes.JuliaType(::Val{:A0}) = A0
ArrowTypes.arrowname(::Type{A1}) = :A1
ArrowTypes.JuliaType(::Val{:A1}) = A1
ArrowTypes.arrowname(::Type{A2}) = :A2
ArrowTypes.JuliaType(::Val{:A2}) = A2
struct MyUnion{T <:Tuple}
elts::T
end
ArrowTypes.arrowname(::Type{<:MyUnion}) = :MyUnion
ArrowTypes.JuliaType(::Val{:MyUnion}) = MyUnion
ArrowTypes.ArrowType(::Type{<:MyUnion}) = ArrowTypes.UnionKind()
ArrowTypes.toarrow(u::MyUnion{T}) where {T} = collect(Union{T.parameters...}, u.elts)
ArrowTypes.fromarrow(::Type{<:MyUnion}, args...) = MyUnion(args)
```
Then:
```julia
julia> u = MyUnion((A0(), A1(1), A2(1, 2.0)))
MyUnion{Tuple{A0, A1, A2}}((A0(), A1(1), A2(1, 2.0)))
julia> ArrowTypes.toarrow(u)
3-element Vector{Union{A0, A1, A2}}:
A0()
A1(1)
A2(1, 2.0)
julia> tbl = (; col = [u]);
julia> Arrow.Table(Arrow.tobuffer(tbl)).col[1]
ERROR: MethodError: no method matching isstringtype(::ArrowTypes.StructKind)
Closest candidates are:
isstringtype(::ArrowTypes.ListKind{stringtype}) where stringtype at ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:196
isstringtype(::Type{ArrowTypes.ListKind{stringtype}}) where stringtype at ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:197
Stacktrace:
[1] getindex(l::Arrow.List{MyUnion, Int32, Arrow.DenseUnion{Union{Missing, A0, A1, A2}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Dense, nothing, Tuple{Union{Missing, A0, A1}, A2}}, Tuple{Arrow.DenseUnion{Union{Missing, A0, A1}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Dense, nothing, Tuple{Union{Missing, A0}, A1}}, Tuple{Arrow.Struct{Union{Missing, A0}, Tuple{}}, Arrow.Struct{A1, Tuple{Arrow.Primitive{Int64, Vector{Int64}}}}}}, Arrow.Struct{A2, Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64, Vector{Float64}}}}}}}, i::Int64)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:52
[2] top-level scope
@ REPL[25]:1
```
I took a guess and added
```julia
ArrowTypes.isstringtype(::ArrowTypes.StructKind) = false
```
which seems to fix it
```julia
julia> Arrow.Table(Arrow.tobuffer(tbl)).col[1]
MyUnion{Tuple{SubArray{Union{Missing, A0, A1, A2}, 1, Arrow.DenseUnion{Union{Missing, A0, A1, A2}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Dense, nothing, Tuple{Union{Missing, A0, A1}, A2}}, Tuple{Arrow.DenseUnion{Union{Missing, A0, A1}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Dense, nothing, Tuple{Union{Missing, A0}, A1}}, Tuple{Arrow.Struct{Union{Missing, A0}, Tuple{}}, Arrow.Struct{A1, Tuple{Arrow.Primitive{Int64, Vector{Int64}}}}}}, Arrow.Struct{A2, Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64, Vector{Float64}}}}}}, Tuple{UnitRange{Int64}}, true}}}((Union{Missing, A0, A1, A2}[A0(), A1(1), A2(1, 2.0)],))
```
However, in my real code, I was using this macro to define the methods:
```julia
macro arrow_record(T1)
T = esc(T1)
name = :(Symbol("JuliaLang.", string(parentmodule($T)), '.', string(nameof($T))))
return quote
ArrowTypes.arrowname(::Type{$T}) = $name
ArrowTypes.ArrowType(::Type{$T}) = fieldtypes($T)
ArrowTypes.toarrow(obj::$T) = ntuple(i -> getfield(obj, i), fieldcount($T))
ArrowTypes.JuliaType(::Val{$name}, ::Any) = $T
ArrowTypes.fromarrow(::Type{$T}, args) = $T(args...)
ArrowTypes.fromarrow(::Type{$T}, arg::$T) = arg
end
end
```
and had a second `A2`-style struct,
```julia
struct A22
x::Int
y::Float64
end
```
If I do
```julia
@arrow_record A2
@arrow_record A22
```
and define
```
julia> u1 = MyUnion((A2(1, 2.0), A22(2, 3.0)))
MyUnion{Tuple{A2, A22}}((A2(1, 2.0), A22(2, 3.0)))
julia> u2 = MyUnion((A22(1, 2.0), A2(2, 3.0)))
MyUnion{Tuple{A22, A2}}((A22(1, 2.0), A2(2, 3.0)))
julia> tbl = (; col = [u1, u2]);
```
Then I get
```julia
julia> Arrow.Table(Arrow.tobuffer(tbl)).col[1]
ERROR: TypeError: in Union, expected Type, got a value of type Tuple{DataType, DataType}
Stacktrace:
[1] ArrowType(#unused#::Type{Union{Missing, A2}})
@ ArrowTypes ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:71
[2] ArrowTypes.ToArrow(x::Vector{Union{Missing, A2}})
@ ArrowTypes ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:338
[3] arrowvector(x::Vector{Union{Missing, A2}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:61
[4] #56
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:220 [inlined]
[5] iterate
@ ./generator.jl:47 [inlined]
[6] collect(itr::Base.Generator{Base.Iterators.Enumerate{Tuple{Vector{Union{Missing, A2}}, Vector{A22}}}, Arrow.var"#56#62"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing, Bool}}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}}})
@ Base ./array.jl:787
[7] map(f::Function, A::Base.Iterators.Enumerate{Tuple{Vector{Union{Missing, A2}}, Vector{A22}}})
@ Base ./abstractarray.jl:2961
[8] arrowvector(::ArrowTypes.UnionKind, x::Arrow.DenseUnionVector{Arrow.ToList{Union{A2, A22}, false, Vector{Union{A2, A22}}, Int32}, Tuple{A2, A22}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:220
[9] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[10] arrowvector(x::Arrow.DenseUnionVector{Arrow.ToList{Union{A2, A22}, false, Vector{Union{A2, A22}}, Int32}, Tuple{A2, A22}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[11] arrowvector(U::Union, x::Arrow.ToList{Union{A2, A22}, false, Vector{Union{A2, A22}}, Int32}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; denseunions::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :maxdepth, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:210
[12] arrowvector(x::Arrow.ToList{Union{A2, A22}, false, Vector{Union{A2, A22}}, Int32}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[13] arrowvector(::ArrowTypes.ListKind{false}, x::ArrowTypes.ToArrow{Vector{Union{A2, A22}}, Vector{MyUnion}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Base.ImmutableDict{String, String}; largelists::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:198
[14] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[15] arrowvector(x::Vector{MyUnion}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists, :denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[16] toarrowvector(x::Vector{MyUnion}, i::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing, kw::Base.Pairs{Symbol, Integer, NTuple{5, Symbol}, NamedTuple{(:largelists, :denseunions, :dictencode, :dictencodenested, :maxdepth), Tuple{Bool, Bool, Bool, Bool, Int64}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:36
[17] (::Arrow.var"#139#140"{Dict{Int64, Any}, Bool, Nothing, Bool, Bool, Bool, Int64, Nothing, Vector{Arrow.DictEncoding}, Vector{Type}, Vector{Any}})(col::Vector{MyUnion}, i::Int64, nm::Symbol)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:331
[18] eachcolumn
@ ~/.julia/packages/Tables/PxO1m/src/utils.jl:70 [inlined]
[19] toarrowtable(cols::NamedTuple{(:col,), Tuple{Vector{MyUnion}}}, dictencodings::Dict{Int64, Any}, largelists::Bool, compress::Nothing, denseunions::Bool, dictencode::Bool, dictencodenested::Bool, maxdepth::Int64, meta::Nothing, colmeta::Nothing)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:328
[20] macro expansion
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:196 [inlined]
[21] macro expansion
@ ./task.jl:454 [inlined]
[22] write(writer::Arrow.Writer{IOBuffer}, source::NamedTuple{(:col,), Tuple{Vector{MyUnion}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:186
[23] (::Arrow.var"#135#136"{NamedTuple{(:col,), Tuple{Vector{MyUnion}}}})(writer::Arrow.Writer{IOBuffer})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:284
[24] open(::Arrow.var"#135#136"{NamedTuple{(:col,), Tuple{Vector{MyUnion}}}}, ::Type, ::Vararg{Any}; kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:file,), Tuple{Bool}}})
@ Base ./io.jl:384
[25] #write#134
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:283 [inlined]
[26] write
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:282 [inlined]
[27] tobuffer(data::NamedTuple{(:col,), Tuple{Vector{MyUnion}}}; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/utils.jl:193
[28] tobuffer(data::NamedTuple{(:col,), Tuple{Vector{MyUnion}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/utils.jl:191
[29] top-level scope
@ REPL[87]:1
```
So then I turned off dense unions,
```julia
julia> Arrow.Table(Arrow.tobuffer(tbl; denseunions=false)).col[1]
ERROR: MethodError: no method matching zero(::Type{A2})
Closest candidates are:
zero(::Union{Type{P}, P}) where P<:Dates.Period at ~/.asdf/installs/julia/1.8.0/share/julia/stdlib/v1.8/Dates/src/periods.jl:53
zero(::AbstractIrrational) at irrationals.jl:150
zero(::SparseArrays.AbstractSparseArray) at ~/.asdf/installs/julia/1.8.0/share/julia/stdlib/v1.8/SparseArrays/src/SparseArrays.jl:42
...
Stacktrace:
[1] default(T::Type)
@ ArrowTypes ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:305
[2] getindex(A::Arrow.ToSparseUnion{A2, Arrow.ToList{Union{A2, A22}, false, Vector{Union{A2, A22}}, Int32}}, i::Int64)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:160
...
```
So then I added that,
```julia
julia> ArrowTypes.default(::Type{A2}) = A2(1,2)
julia> ArrowTypes.default(::Type{A22}) = A22(1,2)
julia> Arrow.Table(Arrow.tobuffer(tbl; denseunions=false)).col[1]
MyUnion{Tuple{SubArray{Union{A2, A22}, 1, Arrow.SparseUnion{Union{A2, A22}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse, nothing, Tuple{A2, A22}}, Tuple{Arrow.Struct{A2, Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64, Vector{Float64}}}}, Arrow.Struct{A22, Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64, Vector{Float64}}}}}}, Tuple{UnitRange{Int64}}, true}}}((Union{A2, A22}[A2(1, 2.0), A22(2, 3.0)],))
```
and it seems to work!
Ok great. But then in the real code, I have an abstract supertype: (starting a new Julia session now!)
```julia
using Arrow
ArrowTypes.isstringtype(::ArrowTypes.StructKind) = false
abstract type A end
struct A0 <: A end
struct A1 <: A
x::Int
end
struct A2 <: A
x::Int
y::Float64
end
ArrowTypes.arrowname(::Type{A0}) = :A0
ArrowTypes.JuliaType(::Val{:A0}) = A0
ArrowTypes.default(::Type{A0}) = A0()
ArrowTypes.arrowname(::Type{A1}) = :A1
ArrowTypes.JuliaType(::Val{:A1}) = A1
ArrowTypes.default(::Type{A1}) = A1(1)
ArrowTypes.arrowname(::Type{A2}) = :A2
ArrowTypes.JuliaType(::Val{:A2}) = A2
ArrowTypes.default(::Type{A2}) = A2(1, 2)
struct MyUnion{T<:NTuple{N,A} where {N}} <: A
elts::T
end
ArrowTypes.arrowname(::Type{<:MyUnion}) = :MyUnion
ArrowTypes.JuliaType(::Val{:MyUnion}) = MyUnion
ArrowTypes.ArrowType(::Type{<:MyUnion}) = ArrowTypes.UnionKind()
ArrowTypes.toarrow(u::MyUnion{T}) where {T} = collect(Union{T.parameters...}, u.elts)
ArrowTypes.fromarrow(::Type{<:MyUnion}, args...) = MyUnion(args)
```
and then I get
```julia
julia> tbl = (; col = [MyUnion((A0(), A2(-1, 1))), MyUnion((A2(-2, 2),))])
(col = MyUnion[MyUnion{Tuple{A0, A2}}((A0(), A2(-1, 1.0))), MyUnion{Tuple{A2}}((A2(-2, 2.0),))],)
julia> Arrow.Table(Arrow.tobuffer(tbl; denseunions=false)).col[1]
ERROR: MethodError: no method matching MyUnion(::Tuple{SubArray{Union{A0, A2}, 1, Arrow.SparseUnion{Union{A0, A2}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse, nothing, Tuple{A0, A2}}, Tuple{Arrow.Struct{A0, Tuple{}}, Arrow.Struct{A2, Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64, Vector{Float64}}}}}}, Tuple{UnitRange{Int64}}, true}})
Closest candidates are:
MyUnion(::T) where T<:(Tuple{Vararg{A, N}} where N) at REPL[40]:2
Stacktrace:
[1] fromarrow(#unused#::Type{MyUnion}, args::SubArray{Union{A0, A2}, 1, Arrow.SparseUnion{Union{A0, A2}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse, nothing, Tuple{A0, A2}}, Tuple{Arrow.Struct{A0, Tuple{}}, Arrow.Struct{A2, Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64, Vector{Float64}}}}}}, Tuple{UnitRange{Int64}}, true})
@ Main ./REPL[45]:1
[2] getindex(l::Arrow.List{MyUnion, Int32, Arrow.SparseUnion{Union{A0, A2}, Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse, nothing, Tuple{A0, A2}}, Tuple{Arrow.Struct{A0, Tuple{}}, Arrow.Struct{A2, Tuple{Arrow.Primitive{Int64, Vector{Int64}}, Arrow.Primitive{Float64, Vector{Float64}}}}}}}, i::Int64)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:61
[3] top-level scope
@ REPL[46]:1
```
which to me looks like the `fromarrow` for `MyUnion` is getting called before the arguments are `fromarrow`'d into the right types?
Also, if I add a few more types:
```julia
struct A22 <: A
x::Int
y::Float64
end
ArrowTypes.arrowname(::Type{A22}) = :A22
ArrowTypes.JuliaType(::Val{:A22}) = A22
ArrowTypes.default(::Type{A22}) = A22(1, 2)
struct A23 <: A
x::Int
y::Float64
end
ArrowTypes.arrowname(::Type{A23}) = :A23
ArrowTypes.JuliaType(::Val{:A23}) = A23
ArrowTypes.default(::Type{A23}) = A23(1, 2)
tbl = (; col = [MyUnion((A2(1, 1), A22(-2, 2), A0())), MyUnion((A22(0, 0), A22(1, 1), A23(1, 2)))])
```
I get
```julia
julia> Arrow.tobuffer(tbl; denseunions=false)
ERROR: MethodError: no method matching zero(::Type{Union{A0, A2, A22}})
Closest candidates are:
zero(::Union{Type{P}, P}) where P<:Dates.Period at ~/.asdf/installs/julia/1.8.0/share/julia/stdlib/v1.8/Dates/src/periods.jl:53
zero(::AbstractIrrational) at irrationals.jl:150
zero(::SparseArrays.AbstractSparseArray) at ~/.asdf/installs/julia/1.8.0/share/julia/stdlib/v1.8/SparseArrays/src/SparseArrays.jl:42
...
Stacktrace:
[1] default(T::Type)
@ ArrowTypes ~/.julia/packages/ArrowTypes/dkiHE/src/ArrowTypes.jl:305
[2] getindex
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:160 [inlined]
[3] iterate(A::Arrow.ToSparseUnion{Union{A0, A2, A22}, ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}}, state::Tuple{Base.OneTo{Int64}, Int64})
@ Base ./abstractarray.jl:1167
[4] iterate
@ ./iterators.jl:167 [inlined]
[5] sparsetypeids(#unused#::Type{Arrow.UnionT{Arrow.Flatbuf.UnionModes.Sparse, nothing, Tuple{Union{A0, A2}, A22}}}, x::Arrow.ToSparseUnion{Union{A0, A2, A22}, ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:144
[6] arrowvector(::ArrowTypes.UnionKind, x::Arrow.SparseUnionVector{Arrow.ToSparseUnion{Union{A0, A2, A22}, ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}}, Tuple{Union{A0, A2}, A22}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:225
[7] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[8] arrowvector(x::Arrow.SparseUnionVector{Arrow.ToSparseUnion{Union{A0, A2, A22}, ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}}, Tuple{Union{A0, A2}, A22}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[9] arrowvector(U::Union, x::Arrow.ToSparseUnion{Union{A0, A2, A22}, ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; denseunions::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :maxdepth, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:210
[10] arrowvector(x::Arrow.ToSparseUnion{Union{A0, A2, A22}, ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[11] (::Arrow.var"#59#65"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing, Bool}}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}, DataType})(j::Int64)
@ Arrow ./none:0
[12] iterate
@ ./generator.jl:47 [inlined]
[13] collect(itr::Base.Generator{UnitRange{Int64}, Arrow.var"#59#65"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing, Bool}}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}, DataType}})
@ Base ./array.jl:787
[14] _totuple
@ ./tuple.jl:349 [inlined]
[15] Tuple(itr::Base.Generator{UnitRange{Int64}, Arrow.var"#59#65"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing, Bool}}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}, DataType}})
@ Base ./tuple.jl:317
[16] arrowvector(::ArrowTypes.UnionKind, x::Arrow.SparseUnionVector{ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}, Tuple{Union{A0, A2, A22}, A23}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:226
[17] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[18] arrowvector(x::Arrow.SparseUnionVector{ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}, Tuple{Union{A0, A2, A22}, A23}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:denseunions, :lareglists, :compression, :dictencodenested), Tuple{Bool, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[19] arrowvector(U::Union, x::ArrowTypes.ToArrow{Union{A0, A2, A22, A23}, Arrow.ToList{Any, false, Vector, Int32}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; denseunions::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :maxdepth, :lareglists, :compression, :dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/unions.jl:210
[20] arrowvector(x::Arrow.ToList{Any, false, Vector, Int32}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[21] arrowvector(::ArrowTypes.ListKind{false}, x::ArrowTypes.ToArrow{Vector, Vector{MyUnion}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Base.ImmutableDict{String, String}; largelists::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:198
[22] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[23] arrowvector(x::Vector{MyUnion}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists, :denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[24] toarrowvector(x::Vector{MyUnion}, i::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing, kw::Base.Pairs{Symbol, Integer, NTuple{5, Symbol}, NamedTuple{(:largelists, :denseunions, :dictencode, :dictencodenested, :maxdepth), Tuple{Bool, Bool, Bool, Bool, Int64}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:36
[25] (::Arrow.var"#139#140"{Dict{Int64, Any}, Bool, Nothing, Bool, Bool, Bool, Int64, Nothing, Vector{Arrow.DictEncoding}, Vector{Type}, Vector{Any}})(col::Vector{MyUnion}, i::Int64, nm::Symbol)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:331
[26] eachcolumn
@ ~/.julia/packages/Tables/PxO1m/src/utils.jl:70 [inlined]
[27] toarrowtable(cols::NamedTuple{(:col,), Tuple{Vector{MyUnion}}}, dictencodings::Dict{Int64, Any}, largelists::Bool, compress::Nothing, denseunions::Bool, dictencode::Bool, dictencodenested::Bool, maxdepth::Int64, meta::Nothing, colmeta::Nothing)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:328
[28] macro expansion
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:196 [inlined]
[29] macro expansion
@ ./task.jl:454 [inlined]
[30] write(writer::Arrow.Writer{IOBuffer}, source::NamedTuple{(:col,), Tuple{Vector{MyUnion}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:186
[31] (::Arrow.var"#135#136"{NamedTuple{(:col,), Tuple{Vector{MyUnion}}}})(writer::Arrow.Writer{IOBuffer})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:284
[32] open(::Arrow.var"#135#136"{NamedTuple{(:col,), Tuple{Vector{MyUnion}}}}, ::Type, ::Vararg{Any}; kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol, Symbol}, NamedTuple{(:file, :denseunions), Tuple{Bool, Bool}}})
@ Base ./io.jl:384
[33] #write#134
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:283 [inlined]
[34] tobuffer(data::NamedTuple{(:col,), Tuple{Vector{MyUnion}}}; kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:denseunions,), Tuple{Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/utils.jl:193
[35] top-level scope
@ REPL[53]:1
```
Then if I `@arrow_record` them, I get a different error:
```julia
julia> macro arrow_record(T1)
T = esc(T1)
name = :(Symbol("JuliaLang.", string(parentmodule($T)), '.', string(nameof($T))))
return quote
ArrowTypes.arrowname(::Type{$T}) = $name
ArrowTypes.ArrowType(::Type{$T}) = fieldtypes($T)
ArrowTypes.toarrow(obj::$T) = ntuple(i -> getfield(obj, i), fieldcount($T))
ArrowTypes.JuliaType(::Val{$name}, ::Any) = $T
ArrowTypes.fromarrow(::Type{$T}, args) = $T(args...)
ArrowTypes.fromarrow(::Type{$T}, arg::$T) = arg
end
end
@arrow_record (macro with 1 method)
julia> @arrow_record A0
julia> @arrow_record A2
julia> @arrow_record A22
julia> @arrow_record A23
julia> @arrow_record A1
julia> Arrow.tobuffer(tbl; denseunions=false)
ERROR: ArgumentError: type does not have a definite number of fields
Stacktrace:
[1] fieldcount(t::Any)
@ Base ./reflection.jl:804
[2] arrowvector(::ArrowTypes.StructKind, x::ArrowTypes.ToArrow{Tuple{Vararg{Real}}, Arrow.ToList{Any, false, Vector, Int32}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/struct.jl:93
[3] arrowvector(::Type{Tuple{Vararg{Real}}}, x::ArrowTypes.ToArrow{Tuple{Vararg{Real}}, Arrow.ToList{Any, false, Vector, Int32}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Int64, Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90
[4] arrowvector(x::Arrow.ToList{Any, false, Vector, Int32}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[5] arrowvector(::ArrowTypes.ListKind{false}, x::ArrowTypes.ToArrow{Vector, Vector{MyUnion}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Base.ImmutableDict{String, String}; largelists::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/list.jl:198
[6] #arrowvector#10
@ ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:90 [inlined]
[7] arrowvector(x::Vector{MyUnion}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists, :denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:67
[8] toarrowvector(x::Vector{MyUnion}, i::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing, kw::Base.Pairs{Symbol, Integer, NTuple{5, Symbol}, NamedTuple{(:largelists, :denseunions, :dictencode, :dictencodenested, :maxdepth), Tuple{Bool, Bool, Bool, Bool, Int64}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/arraytypes/arraytypes.jl:36
[9] (::Arrow.var"#139#140"{Dict{Int64, Any}, Bool, Nothing, Bool, Bool, Bool, Int64, Nothing, Vector{Arrow.DictEncoding}, Vector{Type}, Vector{Any}})(col::Vector{MyUnion}, i::Int64, nm::Symbol)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:331
[10] eachcolumn
@ ~/.julia/packages/Tables/PxO1m/src/utils.jl:70 [inlined]
[11] toarrowtable(cols::NamedTuple{(:col,), Tuple{Vector{MyUnion}}}, dictencodings::Dict{Int64, Any}, largelists::Bool, compress::Nothing, denseunions::Bool, dictencode::Bool, dictencodenested::Bool, maxdepth::Int64, meta::Nothing, colmeta::Nothing)
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:328
[12] macro expansion
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:196 [inlined]
[13] macro expansion
@ ./task.jl:454 [inlined]
[14] write(writer::Arrow.Writer{IOBuffer}, source::NamedTuple{(:col,), Tuple{Vector{MyUnion}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:186
[15] (::Arrow.var"#135#136"{NamedTuple{(:col,), Tuple{Vector{MyUnion}}}})(writer::Arrow.Writer{IOBuffer})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/write.jl:284
[16] open(::Arrow.var"#135#136"{NamedTuple{(:col,), Tuple{Vector{MyUnion}}}}, ::Type, ::Vararg{Any}; kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol, Symbol}, NamedTuple{(:file, :denseunions), Tuple{Bool, Bool}}})
@ Base ./io.jl:384
[17] #write#134
@ ~/.julia/packages/Arrow/ZlMFU/src/write.jl:283 [inlined]
[18] tobuffer(data::NamedTuple{(:col,), Tuple{Vector{MyUnion}}}; kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:denseunions,), Tuple{Bool}}})
@ Arrow ~/.julia/packages/Arrow/ZlMFU/src/utils.jl:193
[19] top-level scope
@ REPL[63]:1
```
(Note that one gives the same error if `denseunions=true`).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@arrow.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org