You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2022/07/11 23:50:06 UTC
[arrow-datafusion] branch master updated: Update to arrow 18.0.0 (#2856)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new d0d5564b8 Update to arrow 18.0.0 (#2856)
d0d5564b8 is described below
commit d0d5564b8f689a01e542b8c1df829d74d0fab2b0
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Mon Jul 11 19:50:02 2022 -0400
Update to arrow 18.0.0 (#2856)
* Update to arrow/parquet 18.0.0
* clean up field creation API
* Update use `BasicDecimalArray`
---
datafusion-cli/Cargo.toml | 2 +-
datafusion-examples/Cargo.toml | 2 +-
datafusion/common/Cargo.toml | 4 ++--
datafusion/core/Cargo.toml | 4 ++--
datafusion/core/fuzz-utils/Cargo.toml | 2 +-
datafusion/core/src/physical_plan/hash_join.rs | 10 +++++-----
datafusion/core/src/physical_plan/hash_utils.rs | 6 +++---
datafusion/core/src/physical_plan/join_utils.rs | 3 +--
datafusion/expr/Cargo.toml | 2 +-
datafusion/jit/Cargo.toml | 2 +-
datafusion/optimizer/Cargo.toml | 2 +-
datafusion/physical-expr/Cargo.toml | 2 +-
datafusion/physical-expr/src/aggregate/min_max.rs | 9 +++++----
datafusion/physical-expr/src/aggregate/sum.rs | 4 ++--
datafusion/physical-expr/src/expressions/cast.rs | 6 +++---
datafusion/physical-expr/src/expressions/try_cast.rs | 3 ++-
datafusion/proto/Cargo.toml | 2 +-
datafusion/row/Cargo.toml | 2 +-
datafusion/sql/Cargo.toml | 2 +-
19 files changed, 35 insertions(+), 34 deletions(-)
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index b7755b3d9..3f1a7a369 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.59"
readme = "README.md"
[dependencies]
-arrow = { version = "17.0.0" }
+arrow = { version = "18.0.0" }
clap = { version = "3", features = ["derive", "cargo"] }
datafusion = { path = "../datafusion/core", version = "9.0.0" }
dirs = "4.0.0"
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index 44c40d455..91ff44fe8 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -34,7 +34,7 @@ path = "examples/avro_sql.rs"
required-features = ["datafusion/avro"]
[dev-dependencies]
-arrow-flight = { version = "17.0.0" }
+arrow-flight = { version = "18.0.0" }
async-trait = "0.1.41"
datafusion = { path = "../datafusion/core" }
futures = "0.3"
diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
index 03809bcf5..318281611 100644
--- a/datafusion/common/Cargo.toml
+++ b/datafusion/common/Cargo.toml
@@ -38,11 +38,11 @@ jit = ["cranelift-module"]
pyarrow = ["pyo3"]
[dependencies]
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
cranelift-module = { version = "0.85.0", optional = true }
object_store = { version = "0.3", optional = true }
ordered-float = "3.0"
-parquet = { version = "17.0.0", features = ["arrow"], optional = true }
+parquet = { version = "18.0.0", features = ["arrow"], optional = true }
pyo3 = { version = "0.16", optional = true }
sqlparser = "0.18"
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index c6cf32bd7..fb5e5b3f3 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -55,7 +55,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
bytes = "1.1"
@@ -78,7 +78,7 @@ num_cpus = "1.13.0"
object_store = "0.3.0"
ordered-float = "3.0"
parking_lot = "0.12"
-parquet = { version = "17.0.0", features = ["arrow", "async"] }
+parquet = { version = "18.0.0", features = ["arrow", "async"] }
paste = "^1.0"
pin-project-lite = "^0.2.7"
pyo3 = { version = "0.16", optional = true }
diff --git a/datafusion/core/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml
index 1cc8a60e7..8544d88f1 100644
--- a/datafusion/core/fuzz-utils/Cargo.toml
+++ b/datafusion/core/fuzz-utils/Cargo.toml
@@ -23,6 +23,6 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
env_logger = "0.9.0"
rand = "0.8"
diff --git a/datafusion/core/src/physical_plan/hash_join.rs b/datafusion/core/src/physical_plan/hash_join.rs
index 7d7f1d4f8..6db504034 100644
--- a/datafusion/core/src/physical_plan/hash_join.rs
+++ b/datafusion/core/src/physical_plan/hash_join.rs
@@ -22,11 +22,11 @@ use ahash::RandomState;
use arrow::{
array::{
- as_dictionary_array, as_string_array, ArrayData, ArrayRef, BooleanArray,
- Date32Array, Date64Array, DecimalArray, DictionaryArray, LargeStringArray,
- PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray,
- TimestampSecondArray, UInt32BufferBuilder, UInt32Builder, UInt64BufferBuilder,
- UInt64Builder,
+ as_dictionary_array, as_string_array, ArrayData, ArrayRef, BasicDecimalArray,
+ BooleanArray, Date32Array, Date64Array, DecimalArray, DictionaryArray,
+ LargeStringArray, PrimitiveArray, TimestampMicrosecondArray,
+ TimestampMillisecondArray, TimestampSecondArray, UInt32BufferBuilder,
+ UInt32Builder, UInt64BufferBuilder, UInt64Builder,
},
compute,
datatypes::{
diff --git a/datafusion/core/src/physical_plan/hash_utils.rs b/datafusion/core/src/physical_plan/hash_utils.rs
index 79fdcf99c..81bf5c1f3 100644
--- a/datafusion/core/src/physical_plan/hash_utils.rs
+++ b/datafusion/core/src/physical_plan/hash_utils.rs
@@ -20,9 +20,9 @@
use crate::error::{DataFusionError, Result};
use ahash::{CallHasher, RandomState};
use arrow::array::{
- Array, ArrayRef, BooleanArray, Date32Array, Date64Array, DecimalArray,
- DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
- Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray,
+ Array, ArrayRef, BasicDecimalArray, BooleanArray, Date32Array, Date64Array,
+ DecimalArray, DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array,
+ Int64Array, Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray,
TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,
UInt16Array, UInt32Array, UInt64Array, UInt8Array,
};
diff --git a/datafusion/core/src/physical_plan/join_utils.rs b/datafusion/core/src/physical_plan/join_utils.rs
index 1c0440db8..cbabf548c 100644
--- a/datafusion/core/src/physical_plan/join_utils.rs
+++ b/datafusion/core/src/physical_plan/join_utils.rs
@@ -170,8 +170,7 @@ fn output_join_field(old_field: &Field, join_type: &JoinType, is_left: bool) ->
};
if force_nullable {
- // Could cleanup after https://github.com/apache/arrow-rs/issues/1934
- Field::new(old_field.name(), old_field.data_type().clone(), true)
+ old_field.clone().with_nullable(true)
} else {
old_field.clone()
}
diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
index 4f8f1e01a..e0d1e0827 100644
--- a/datafusion/expr/Cargo.toml
+++ b/datafusion/expr/Cargo.toml
@@ -36,6 +36,6 @@ path = "src/lib.rs"
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "9.0.0" }
sqlparser = "0.18"
diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml
index 8ffdf266c..8cff5d646 100644
--- a/datafusion/jit/Cargo.toml
+++ b/datafusion/jit/Cargo.toml
@@ -36,7 +36,7 @@ path = "src/lib.rs"
jit = []
[dependencies]
-arrow = { version = "17.0.0" }
+arrow = { version = "18.0.0" }
cranelift = "0.85.0"
cranelift-jit = "0.85.0"
cranelift-module = "0.85.0"
diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml
index 1104cbe81..0efd7981c 100644
--- a/datafusion/optimizer/Cargo.toml
+++ b/datafusion/optimizer/Cargo.toml
@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []
[dependencies]
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
chrono = { version = "0.4", default-features = false }
datafusion-common = { path = "../common", version = "9.0.0" }
diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
index 41a16f21c..0c7bffdb7 100644
--- a/datafusion/physical-expr/Cargo.toml
+++ b/datafusion/physical-expr/Cargo.toml
@@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"]
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
chrono = { version = "0.4", default-features = false }
diff --git a/datafusion/physical-expr/src/aggregate/min_max.rs b/datafusion/physical-expr/src/aggregate/min_max.rs
index 37df7713e..811aa473c 100644
--- a/datafusion/physical-expr/src/aggregate/min_max.rs
+++ b/datafusion/physical-expr/src/aggregate/min_max.rs
@@ -26,10 +26,11 @@ use arrow::compute;
use arrow::datatypes::{DataType, TimeUnit};
use arrow::{
array::{
- ArrayRef, Date32Array, Date64Array, Float32Array, Float64Array, Int16Array,
- Int32Array, Int64Array, Int8Array, LargeStringArray, StringArray,
- TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
- TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+ ArrayRef, BasicDecimalArray, Date32Array, Date64Array, Float32Array,
+ Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, LargeStringArray,
+ StringArray, TimestampMicrosecondArray, TimestampMillisecondArray,
+ TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array,
+ UInt64Array, UInt8Array,
},
datatypes::Field,
};
diff --git a/datafusion/physical-expr/src/aggregate/sum.rs b/datafusion/physical-expr/src/aggregate/sum.rs
index 056af7747..ee8b224fe 100644
--- a/datafusion/physical-expr/src/aggregate/sum.rs
+++ b/datafusion/physical-expr/src/aggregate/sum.rs
@@ -26,8 +26,8 @@ use arrow::compute;
use arrow::datatypes::DataType;
use arrow::{
array::{
- ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
- Int8Array, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+ ArrayRef, BasicDecimalArray, Float32Array, Float64Array, Int16Array, Int32Array,
+ Int64Array, Int8Array, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
},
datatypes::Field,
};
diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs
index 0108f7fee..de1909676 100644
--- a/datafusion/physical-expr/src/expressions/cast.rs
+++ b/datafusion/physical-expr/src/expressions/cast.rs
@@ -161,9 +161,9 @@ mod tests {
use crate::expressions::col;
use arrow::{
array::{
- Array, DecimalArray, Float32Array, Float64Array, Int16Array, Int32Array,
- Int64Array, Int8Array, StringArray, Time64NanosecondArray,
- TimestampNanosecondArray, UInt32Array,
+ Array, BasicDecimalArray, DecimalArray, Float32Array, Float64Array,
+ Int16Array, Int32Array, Int64Array, Int8Array, StringArray,
+ Time64NanosecondArray, TimestampNanosecondArray, UInt32Array,
},
datatypes::*,
util::decimal::{BasicDecimal, Decimal128},
diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs
index c5178b4db..2509c1d6b 100644
--- a/datafusion/physical-expr/src/expressions/try_cast.rs
+++ b/datafusion/physical-expr/src/expressions/try_cast.rs
@@ -119,7 +119,8 @@ mod tests {
use super::*;
use crate::expressions::col;
use arrow::array::{
- DecimalArray, DecimalBuilder, StringArray, Time64NanosecondArray,
+ BasicDecimalArray, DecimalArray, DecimalBuilder, StringArray,
+ Time64NanosecondArray,
};
use arrow::util::decimal::{BasicDecimal, Decimal128};
use arrow::{
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index d4ee90101..ff6da23bd 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -35,7 +35,7 @@ path = "src/lib.rs"
[features]
[dependencies]
-arrow = { version = "17.0.0" }
+arrow = { version = "18.0.0" }
datafusion = { path = "../core", version = "9.0.0" }
datafusion-common = { path = "../common", version = "9.0.0" }
datafusion-expr = { path = "../expr", version = "9.0.0" }
diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml
index 0dd07a5e2..b213f7870 100644
--- a/datafusion/row/Cargo.toml
+++ b/datafusion/row/Cargo.toml
@@ -37,7 +37,7 @@ path = "src/lib.rs"
jit = ["datafusion-jit"]
[dependencies]
-arrow = { version = "17.0.0" }
+arrow = { version = "18.0.0" }
datafusion-common = { path = "../common", version = "9.0.0" }
datafusion-jit = { path = "../jit", version = "9.0.0", optional = true }
paste = "^1.0"
diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml
index fa477a0b7..4258046e7 100644
--- a/datafusion/sql/Cargo.toml
+++ b/datafusion/sql/Cargo.toml
@@ -38,7 +38,7 @@ unicode_expressions = []
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "17.0.0", features = ["prettyprint"] }
+arrow = { version = "18.0.0", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "9.0.0" }
datafusion-expr = { path = "../expr", version = "9.0.0" }
hashbrown = "0.12"