You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@arrow.apache.org by "ericphanson (via GitHub)" <gi...@apache.org> on 2023/06/11 00:57:59 UTC
[GitHub] [arrow-julia] ericphanson opened a new issue, #461: Issue with `Union{Missing, VersionNumber}`
ericphanson opened a new issue, #461:
URL: https://github.com/apache/arrow-julia/issues/461
```julia
struct A
v::Union{Missing, VersionNumber}
end
table = [A(v"1"), A(missing)]
```
gives
```julia
julia> Arrow.tobuffer(table)
ERROR: ArgumentError: type does not have a definite number of fields
Stacktrace:
[1] fieldcount(t::Any)
@ Base ./reflection.jl:814
[2] default(#unused#::Type{Tuple{Vararg{Union{UInt64, String}}}})
@ ArrowTypes ~/.julia/packages/ArrowTypes/aTFES/src/ArrowTypes.jl:352
[3] getindex
@ ~/.julia/packages/Arrow/rYdxZ/src/arraytypes/struct.jl:80 [inlined]
[4] ArrowTypes.ToArrow(x::Arrow.ToStruct{Tuple{Vararg{Union{UInt64, String}}}, 4, Vector{Union{Missing, VersionNumber}}})
@ ArrowTypes ~/.julia/packages/ArrowTypes/aTFES/src/ArrowTypes.jl:381
[5] arrowvector(x::Arrow.ToStruct{Tuple{Vararg{Union{UInt64, String}}}, 4, Vector{Union{Missing, VersionNumber}}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists, :denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/arraytypes/arraytypes.jl:62
[6] (::Arrow.var"#49#50"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :largelists, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool, Bool}}}, Vector{Union{Missing, VersionNumber}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}})(j::Int64)
@ Arrow ./none:0
[7] iterate
@ ./generator.jl:47 [inlined]
[8] collect_to!(dest::Vector{Arrow.Primitive{UInt32}}, itr::Base.Generator{UnitRange{Int64}, Arrow.var"#49#50"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :largelists, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool, Bool}}}, Vector{Union{Missing, VersionNumber}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}}}, offs::Int64, st::Int64)
@ Base ./array.jl:840
[9] collect_to!(dest::Vector{Arrow.Primitive{UInt32, Arrow.ToStruct{UInt32, 1, Vector{Union{Missing, VersionNumber}}}}}, itr::Base.Generator{UnitRange{Int64}, Arrow.var"#49#50"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :largelists, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool, Bool}}}, Vector{Union{Missing, VersionNumber}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}}}, offs::Int64, st::Int64)
@ Base ./array.jl:848
[10] collect_to_with_first!(dest::Vector{Arrow.Primitive{UInt32, Arrow.ToStruct{UInt32, 1, Vector{Union{Missing, VersionNumber}}}}}, v1::Arrow.Primitive{UInt32, Arrow.ToStruct{UInt32, 1, Vector{Union{Missing, VersionNumber}}}}, itr::Base.Generator{UnitRange{Int64}, Arrow.var"#49#50"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :largelists, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool, Bool}}}, Vector{Union{Missing, VersionNumber}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}}}, st::Int64)
@ Base ./array.jl:818
[11] collect(itr::Base.Generator{UnitRange{Int64}, Arrow.var"#49#50"{Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :largelists, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool, Bool}}}, Vector{Union{Missing, VersionNumber}}, Int64, Int64, Dict{Int64, Any}, Vector{Arrow.DictEncoding}}})
@ Base ./array.jl:792
[12] _totuple
@ ./tuple.jl:401 [inlined]
[13] Tuple
@ ./tuple.jl:369 [inlined]
[14] arrowvector(::ArrowTypes.StructKind, x::Vector{Union{Missing, VersionNumber}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Base.ImmutableDict{String, String}; kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :largelists, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/arraytypes/struct.jl:93
[15] arrowvector
@ ~/.julia/packages/Arrow/rYdxZ/src/arraytypes/struct.jl:89 [inlined]
[16] arrowvector(::Type{VersionNumber}, x::Vector{Union{Missing, VersionNumber}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Base.ImmutableDict{String, String}; kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{6, Symbol}, NamedTuple{(:dictencode, :maxdepth, :compression, :largelists, :denseunions, :dictencodenested), Tuple{Bool, Int64, Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/arraytypes/arraytypes.jl:91
[17] arrowvector(x::Vector{Union{Missing, VersionNumber}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists, :denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/arraytypes/arraytypes.jl:68
[18] arrowvector
@ ~/.julia/packages/Arrow/rYdxZ/src/arraytypes/arraytypes.jl:52 [inlined]
[19] toarrowvector(x::Vector{Union{Missing, VersionNumber}}, i::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing, kw::Base.Pairs{Symbol, Integer, NTuple{5, Symbol}, NamedTuple{(:largelists, :denseunions, :dictencode, :dictencodenested, :maxdepth), Tuple{Bool, Bool, Bool, Bool, Int64}}})
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/arraytypes/arraytypes.jl:37
[20] toarrowvector
@ ~/.julia/packages/Arrow/rYdxZ/src/arraytypes/arraytypes.jl:34 [inlined]
[21] (::Arrow.var"#145#146"{Dict{Int64, Any}, Bool, Nothing, Bool, Bool, Bool, Int64, Nothing, Vector{Arrow.DictEncoding}, Vector{Type}, Vector{Any}})(col::Vector{Union{Missing, VersionNumber}}, i::Int64, nm::Symbol)
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/write.jl:337
[22] eachcolumn
@ ~/.julia/packages/Tables/AcRIE/src/utils.jl:70 [inlined]
[23] toarrowtable(cols::Tables.CopiedColumns{NamedTuple{(:v,), Tuple{Vector{Union{Missing, VersionNumber}}}}}, dictencodings::Dict{Int64, Any}, largelists::Bool, compress::Nothing, denseunions::Bool, dictencode::Bool, dictencodenested::Bool, maxdepth::Int64, meta::Nothing, colmeta::Nothing)
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/write.jl:334
[24] macro expansion
@ ~/.julia/packages/Arrow/rYdxZ/src/write.jl:195 [inlined]
[25] macro expansion
@ ./task.jl:476 [inlined]
[26] write(writer::Arrow.Writer{IOBuffer}, source::Vector{A})
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/write.jl:185
[27] (::Arrow.var"#137#138"{Vector{A}})(writer::Arrow.Writer{IOBuffer})
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/write.jl:283
[28] open(::Arrow.var"#137#138"{Vector{A}}, ::Type, ::Vararg{Any}; kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:file,), Tuple{Bool}}})
@ Base ./io.jl:395
[29] open
@ ./io.jl:392 [inlined]
[30] #write#136
@ ~/.julia/packages/Arrow/rYdxZ/src/write.jl:282 [inlined]
[31] write
@ ~/.julia/packages/Arrow/rYdxZ/src/write.jl:281 [inlined]
[32] tobuffer(data::Vector{A}; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/utils.jl:132
[33] tobuffer(data::Vector{A})
@ Arrow ~/.julia/packages/Arrow/rYdxZ/src/utils.jl:130
[34] top-level scope
@ REPL[30]:1
[35] top-level scope
@ ~/.julia/juliaup/julia-1.9.1+0.x64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:1416
```
on Arrow 2.5 and 2.6, and Julia 1.8 and 1.9.
This is OK if I replace `VersionNumber` with `String`, for example:
```julia
struct B
v::Union{Missing, String}
end
table = [B("1"), B(missing)]
```
gives
```julia
julia> Arrow.tobuffer(table)
IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=false, size=320, maxsize=Inf, ptr=1, mark=-1)
```
as expected
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@arrow.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-julia] quinnj commented on issue #461: Issue with `Union{Missing, VersionNumber}`
Posted by "quinnj (via GitHub)" <gi...@apache.org>.
quinnj commented on issue #461:
URL: https://github.com/apache/arrow-julia/issues/461#issuecomment-1588241110
Ok, fix is up: https://github.com/apache/arrow-julia/pull/465. I believe this has always been an issue. Kind of a gnarly one, but I think the solution proposed should be robust for structs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-julia] quinnj commented on issue #461: Issue with `Union{Missing, VersionNumber}`
Posted by "quinnj (via GitHub)" <gi...@apache.org>.
quinnj commented on issue #461:
URL: https://github.com/apache/arrow-julia/issues/461#issuecomment-1588470236
Alternative fix is up: https://github.com/apache/arrow-julia/pull/466
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-julia] quinnj commented on issue #461: Issue with `Union{Missing, VersionNumber}`
Posted by "quinnj (via GitHub)" <gi...@apache.org>.
quinnj commented on issue #461:
URL: https://github.com/apache/arrow-julia/issues/461#issuecomment-1588197857
Do we know if this used to work and regressed? Or perhaps just never worked?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-julia] Moelf commented on issue #461: Issue with `Union{Missing, VersionNumber}`
Posted by "Moelf (via GitHub)" <gi...@apache.org>.
Moelf commented on issue #461:
URL: https://github.com/apache/arrow-julia/issues/461#issuecomment-1585982928
https://github.com/JuliaLang/julia/blob/c58e508fddba24a7508b3018aa7f413362b384ca/base/version.jl#L43-L44
technically this is indeed not fixed number of fields
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-julia] ericphanson commented on issue #461: Issue with `Union{Missing, VersionNumber}`
Posted by "ericphanson (via GitHub)" <gi...@apache.org>.
ericphanson commented on issue #461:
URL: https://github.com/apache/arrow-julia/issues/461#issuecomment-1588239222
I’m not sure, I’ve definitely serialized VersionNumbers before, but maybe not with missing. This came up in a new application, it didn’t break a working one.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-julia] ericphanson commented on issue #461: Issue with `Union{Missing, VersionNumber}`
Posted by "ericphanson (via GitHub)" <gi...@apache.org>.
ericphanson commented on issue #461:
URL: https://github.com/apache/arrow-julia/issues/461#issuecomment-1586144301
I'm not sure if that's the issue, as `VersionNumber`'s alone work:
```julia
julia> struct C
v::VersionNumber
end
julia> table = [C(v"1")]
1-element Vector{C}:
C(v"1.0.0")
julia> Arrow.tobuffer(table)
IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=false, size=1144, maxsize=Inf, ptr=1, mark=-1)
```
and e.g. `fieldcount(VersionNumber)` works fine.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-julia] quinnj closed issue #461: Issue with `Union{Missing, VersionNumber}`
Posted by "quinnj (via GitHub)" <gi...@apache.org>.
quinnj closed issue #461: Issue with `Union{Missing, VersionNumber}`
URL: https://github.com/apache/arrow-julia/issues/461
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org