[arrow-julia] branch main updated: Fix issue with missing and non-concrete Arrow types (#371)

[qu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

quinnj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-julia.git


The following commit(s) were added to refs/heads/main by this push:
     new 94ee070  Fix issue with missing and non-concrete Arrow types (#371)
94ee070 is described below

commit 94ee070adcf6d4a7ed3108561ee014702e75b835
Author: Curtis Vogt <cu...@gmail.com>
AuthorDate: Thu Jan 12 00:58:58 2023 -0600

    Fix issue with missing and non-concrete Arrow types (#371)
    
    * Fix issue with missing and non-concrete Arrow types
    
    * Add ArrowTypes test
    
    * Remove Arrow tests that require new version of ArrowTypes (for now)
    
    * Bump ArrowTypes version
    
    * Add test for `Any[missing]`
    
    * Add ToArrow testset
    
    * Make existing ToArrow tests more specific
    
    * fixup! Make existing ToArrow tests more specific
    
    * Test eltype
    
    * Test `Union{ZonedDateTime,Missing}` behavior
---
 src/ArrowTypes/Project.toml      |  3 +--
 src/ArrowTypes/src/ArrowTypes.jl |  3 +++
 src/ArrowTypes/test/tests.jl     | 50 +++++++++++++++++++++++++++++++++++-----
 test/runtests.jl                 |  8 +++++++
 4 files changed, 56 insertions(+), 8 deletions(-)

diff --git a/src/ArrowTypes/Project.toml b/src/ArrowTypes/Project.toml
index 250338c..7c23ce9 100644
--- a/src/ArrowTypes/Project.toml
+++ b/src/ArrowTypes/Project.toml
@@ -18,8 +18,7 @@
 name = "ArrowTypes"
 uuid = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
 authors = ["quinnj <qu...@gmail.com>"]
-version = "2.0.1"
-
+version = "2.0.2"
 
 [deps]
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
diff --git a/src/ArrowTypes/src/ArrowTypes.jl b/src/ArrowTypes/src/ArrowTypes.jl
index 8c25b79..f4f9e39 100644
--- a/src/ArrowTypes/src/ArrowTypes.jl
+++ b/src/ArrowTypes/src/ArrowTypes.jl
@@ -349,6 +349,9 @@ function ToArrow(x::A) where {A}
         for i = 2:length(x)
             @inbounds T = promoteunion(T, typeof(toarrow(x[i])))
         end
+        if T === Missing
+            T = promoteunion(T, typeof(toarrow(default(S))))
+        end
     end
     return ToArrow{T, A}(x)
 end
diff --git a/src/ArrowTypes/test/tests.jl b/src/ArrowTypes/test/tests.jl
index f3c9c57..d1e1914 100644
--- a/src/ArrowTypes/test/tests.jl
+++ b/src/ArrowTypes/test/tests.jl
@@ -144,11 +144,49 @@ v_nt = (major=1, minor=0, patch=0, prerelease=(), build=())
 @test !ArrowTypes.concrete_or_concreteunion(Union{Real, String})
 @test !ArrowTypes.concrete_or_concreteunion(Any)
 
-@test ArrowTypes.ToArrow([1,2,3]) == [1,2,3]
-@test ArrowTypes.ToArrow([:hey, :ho]) == ["hey", "ho"]
-x = ArrowTypes.ToArrow(Any[1, 3.14])
-@test x[1] === 1.0
-@test x[2] === 3.14
-@test ArrowTypes.ToArrow(Any[1, 3.14, "hey"]) == [1.0, 3.14, "hey"]
+@testset "ToArrow" begin
+    x = ArrowTypes.ToArrow([1,2,3])
+    @test x isa Vector{Int}
+    @test x == [1,2,3]
+
+    x = ArrowTypes.ToArrow([:hey, :ho])
+    @test x isa ArrowTypes.ToArrow{String, Vector{Symbol}}
+    @test eltype(x) == String
+    @test x == ["hey", "ho"]
+
+    x = ArrowTypes.ToArrow(Any[1, 3.14])
+    @test x isa ArrowTypes.ToArrow{Float64, Vector{Any}}
+    @test eltype(x) == Float64
+    @test x == [1.0, 3.14]
+
+    x = ArrowTypes.ToArrow(Any[1, 3.14, "hey"])
+    @test x isa ArrowTypes.ToArrow{Union{Float64, String}, Vector{Any}}
+    @test eltype(x) == Union{Float64, String}
+    @test x == [1.0, 3.14, "hey"]
+
+    @testset "respect non-missing type" begin
+        struct DateTimeTZ
+            instant::Int64
+            tz::String
+        end
+
+        struct Timestamp{TZ}
+            x::Int64
+        end
+
+        ArrowTypes.ArrowType(::Type{DateTimeTZ}) = Timestamp
+        ArrowTypes.toarrow(x::DateTimeTZ) = Timestamp{Symbol(x.tz)}(x.instant)
+        ArrowTypes.default(::Type{DateTimeTZ}) = DateTimeTZ(0, "UTC")
+
+        T = Union{DateTimeTZ,Missing}
+        @test !ArrowTypes.concrete_or_concreteunion(ArrowTypes.ArrowType(T))
+        @test eltype(ArrowTypes.ToArrow(T[missing])) == Union{Timestamp{:UTC}, Missing}
+
+        # Works since `ArrowTypes.default(Any) === nothing` and
+        # `ArrowTypes.toarrow(nothing) === missing`. Defining `toarrow(::Nothing) = nothing`
+        # would break this test by returning `Union{Nothing,Missing}`.
+        @test eltype(ArrowTypes.ToArrow(Any[missing])) == Missing
+    end
+end
 
 end
diff --git a/test/runtests.jl b/test/runtests.jl
index b040a4f..c1cceec 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -563,6 +563,14 @@ if pkgversion(ArrowTypes) >= v"2.0.1" # need the ArrowTypes bugfix to pass this
     @test isequal(Arrow.Table(Arrow.tobuffer(table)).col, table.col)
 end
 
+# https://github.com/apache/arrow-julia/issues/367
+if pkgversion(ArrowTypes) >= v"2.0.2"
+    t = (; x=Union{ZonedDateTime,Missing}[missing])
+    a = Arrow.Table(Arrow.tobuffer(t))
+    @test Tables.schema(a) == Tables.schema(t)
+    @test isequal(a.x, t.x)
+end
+
 end # @testset "misc"
 
 end