You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2022/06/14 14:18:54 UTC
[arrow-datafusion] branch master updated: Upgrade to arrow `16.0.0` (#2718)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 5096ff4d6 Upgrade to arrow `16.0.0` (#2718)
5096ff4d6 is described below
commit 5096ff4d6b82c2ea3cee0e5c4e5f2d08b3607d10
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Tue Jun 14 10:18:47 2022 -0400
Upgrade to arrow `16.0.0` (#2718)
* Move from deprecated API
* Update parquet tests
* Update test as now Cast from UTF8 -> Boolean is supported
* Update to arrow 16.0.0 now that it is released
---
datafusion-cli/Cargo.toml | 2 +-
datafusion-examples/Cargo.toml | 2 +-
datafusion/common/Cargo.toml | 4 +--
datafusion/core/Cargo.toml | 4 +--
datafusion/core/fuzz-utils/Cargo.toml | 2 +-
.../core/src/datasource/file_format/parquet.rs | 6 ++--
datafusion/core/src/physical_plan/planner.rs | 36 ++++++++++------------
datafusion/expr/Cargo.toml | 2 +-
datafusion/jit/Cargo.toml | 2 +-
datafusion/optimizer/Cargo.toml | 2 +-
datafusion/physical-expr/Cargo.toml | 2 +-
datafusion/proto/Cargo.toml | 2 +-
datafusion/row/Cargo.toml | 2 +-
datafusion/sql/Cargo.toml | 2 +-
14 files changed, 34 insertions(+), 36 deletions(-)
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index 9bded7936..a3f67368b 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.59"
readme = "README.md"
[dependencies]
-arrow = { version = "15.0.0" }
+arrow = { version = "16.0.0" }
clap = { version = "3", features = ["derive", "cargo"] }
datafusion = { path = "../datafusion/core", version = "9.0.0" }
dirs = "4.0.0"
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index 399248f1d..708cc3cc4 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -34,7 +34,7 @@ path = "examples/avro_sql.rs"
required-features = ["datafusion/avro"]
[dev-dependencies]
-arrow-flight = { version = "15.0.0" }
+arrow-flight = { version = "16.0.0" }
async-trait = "0.1.41"
datafusion = { path = "../datafusion/core" }
futures = "0.3"
diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
index 31968e4fc..117653f4c 100644
--- a/datafusion/common/Cargo.toml
+++ b/datafusion/common/Cargo.toml
@@ -38,10 +38,10 @@ jit = ["cranelift-module"]
pyarrow = ["pyo3"]
[dependencies]
-arrow = { version = "15.0.0", features = ["prettyprint"] }
+arrow = { version = "16.0.0", features = ["prettyprint"] }
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
cranelift-module = { version = "0.84.0", optional = true }
ordered-float = "3.0"
-parquet = { version = "15.0.0", features = ["arrow"], optional = true }
+parquet = { version = "16.0.0", features = ["arrow"], optional = true }
pyo3 = { version = "0.16", optional = true }
sqlparser = "0.18"
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index 3291e5ad0..f45b4eb26 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -55,7 +55,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "15.0.0", features = ["prettyprint"] }
+arrow = { version = "16.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
chrono = { version = "0.4", default-features = false }
@@ -77,7 +77,7 @@ num-traits = { version = "0.2", optional = true }
num_cpus = "1.13.0"
ordered-float = "3.0"
parking_lot = "0.12"
-parquet = { version = "15.0.0", features = ["arrow"] }
+parquet = { version = "16.0.0", features = ["arrow"] }
paste = "^1.0"
pin-project-lite = "^0.2.7"
pyo3 = { version = "0.16", optional = true }
diff --git a/datafusion/core/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml
index aae7c108a..4366e3332 100644
--- a/datafusion/core/fuzz-utils/Cargo.toml
+++ b/datafusion/core/fuzz-utils/Cargo.toml
@@ -23,6 +23,6 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-arrow = { version = "15.0.0", features = ["prettyprint"] }
+arrow = { version = "16.0.0", features = ["prettyprint"] }
env_logger = "0.9.0"
rand = "0.8"
diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rs
index 9cf2b89bd..8825fe7e0 100644
--- a/datafusion/core/src/datasource/file_format/parquet.rs
+++ b/datafusion/core/src/datasource/file_format/parquet.rs
@@ -324,7 +324,7 @@ fn fetch_statistics(
let file_schema = arrow_reader.get_schema()?;
let num_fields = table_schema.fields().len();
let fields = table_schema.fields().to_vec();
- let meta_data = arrow_reader.get_metadata();
+ let meta_data = arrow_reader.metadata();
let mut num_rows = 0;
let mut total_byte_size = 0;
@@ -560,8 +560,8 @@ mod tests {
let _ = collect(exec.clone(), task_ctx.clone()).await?;
let _ = collect(exec_projected.clone(), task_ctx).await?;
- assert_bytes_scanned(exec, 2522);
- assert_bytes_scanned(exec_projected, 1924);
+ assert_bytes_scanned(exec, 1409);
+ assert_bytes_scanned(exec_projected, 811);
Ok(())
}
diff --git a/datafusion/core/src/physical_plan/planner.rs b/datafusion/core/src/physical_plan/planner.rs
index 14cdee301..fa112c82d 100644
--- a/datafusion/core/src/physical_plan/planner.rs
+++ b/datafusion/core/src/physical_plan/planner.rs
@@ -1543,6 +1543,7 @@ fn tuple_err<T, R>(value: (Result<T>, Result<R>)) -> Result<(T, R)> {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::assert_contains;
use crate::execution::context::TaskContext;
use crate::execution::options::CsvReadOptions;
use crate::execution::runtime_env::{RuntimeConfig, RuntimeEnv};
@@ -1826,10 +1827,7 @@ mod tests {
#[tokio::test]
async fn in_list_types() -> Result<()> {
// expression: "a in ('a', 1)"
- let list = vec![
- Expr::Literal(ScalarValue::Utf8(Some("a".to_string()))),
- Expr::Literal(ScalarValue::Int64(Some(1))),
- ];
+ let list = vec![lit("a"), lit(1i64)];
let logical_plan = test_csv_scan()
.await?
// filter clause needs the type coercion rule applied
@@ -1841,11 +1839,9 @@ mod tests {
let expected = "InListExpr { expr: Column { name: \"c1\", index: 0 }, list: [Literal { value: Utf8(\"a\") }, CastExpr { expr: Literal { value: Int64(1) }, cast_type: Utf8, cast_options: CastOptions { safe: false } }], negated: false, set: None }";
assert!(format!("{:?}", execution_plan).contains(expected));
- // expression: "a in (true, 'a')"
- let list = vec![
- Expr::Literal(ScalarValue::Boolean(Some(true))),
- Expr::Literal(ScalarValue::Utf8(Some("a".to_string()))),
- ];
+ // expression: "a in (struct::null, 'a')"
+ let list = vec![struct_literal(), lit("a")];
+
let logical_plan = test_csv_scan()
.await?
// filter clause needs the type coercion rule applied
@@ -1854,20 +1850,22 @@ mod tests {
.build()?;
let execution_plan = plan(&logical_plan).await;
- let expected_error = "Unsupported CAST from Utf8 to Boolean";
- match execution_plan {
- Ok(_) => panic!("Expected planning failure"),
- Err(e) => assert!(
- e.to_string().contains(expected_error),
- "Error '{}' did not contain expected error '{}'",
- e,
- expected_error
- ),
- }
+ let e = execution_plan.unwrap_err().to_string();
+ assert_contains!(&e, "Unsupported CAST from Struct");
+ assert_contains!(&e, "to Boolean");
Ok(())
}
+ /// Return a `null` literal representing a struct type like: `{ a: bool }`
+ fn struct_literal() -> Expr {
+ let struct_literal = ScalarValue::Struct(
+ None,
+ Box::new(vec![Field::new("foo", DataType::Boolean, false)]),
+ );
+ lit(struct_literal)
+ }
+
#[tokio::test]
async fn in_set_test() -> Result<()> {
// OPTIMIZER_INSET_THRESHOLD = 10
diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
index f55417899..228fd1ed9 100644
--- a/datafusion/expr/Cargo.toml
+++ b/datafusion/expr/Cargo.toml
@@ -36,6 +36,6 @@ path = "src/lib.rs"
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "15.0.0", features = ["prettyprint"] }
+arrow = { version = "16.0.0", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "9.0.0" }
sqlparser = "0.18"
diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml
index 60cd1cef8..e1947b155 100644
--- a/datafusion/jit/Cargo.toml
+++ b/datafusion/jit/Cargo.toml
@@ -36,7 +36,7 @@ path = "src/lib.rs"
jit = []
[dependencies]
-arrow = { version = "15.0.0" }
+arrow = { version = "16.0.0" }
cranelift = "0.84.0"
cranelift-jit = "0.84.0"
cranelift-module = "0.84.0"
diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml
index d21ac4157..4bd386879 100644
--- a/datafusion/optimizer/Cargo.toml
+++ b/datafusion/optimizer/Cargo.toml
@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []
[dependencies]
-arrow = { version = "15.0.0", features = ["prettyprint"] }
+arrow = { version = "16.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
chrono = { version = "0.4", default-features = false }
datafusion-common = { path = "../common", version = "9.0.0" }
diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
index 7a66a007f..1bb05387b 100644
--- a/datafusion/physical-expr/Cargo.toml
+++ b/datafusion/physical-expr/Cargo.toml
@@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"]
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "15.0.0", features = ["prettyprint"] }
+arrow = { version = "16.0.0", features = ["prettyprint"] }
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
chrono = { version = "0.4", default-features = false }
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index b4e2ba63f..d159e9e79 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -35,7 +35,7 @@ path = "src/lib.rs"
[features]
[dependencies]
-arrow = { version = "15.0.0" }
+arrow = { version = "16.0.0" }
datafusion = { path = "../core", version = "9.0.0" }
datafusion-common = { path = "../common", version = "9.0.0" }
datafusion-expr = { path = "../expr", version = "9.0.0" }
diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml
index 806c05ddb..8bb1e6274 100644
--- a/datafusion/row/Cargo.toml
+++ b/datafusion/row/Cargo.toml
@@ -37,7 +37,7 @@ path = "src/lib.rs"
jit = ["datafusion-jit"]
[dependencies]
-arrow = { version = "15.0.0" }
+arrow = { version = "16.0.0" }
datafusion-common = { path = "../common", version = "9.0.0" }
datafusion-jit = { path = "../jit", version = "9.0.0", optional = true }
paste = "^1.0"
diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml
index 2b6c4df76..42ca5b41c 100644
--- a/datafusion/sql/Cargo.toml
+++ b/datafusion/sql/Cargo.toml
@@ -38,7 +38,7 @@ unicode_expressions = []
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "15.0.0", features = ["prettyprint"] }
+arrow = { version = "16.0.0", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "9.0.0" }
datafusion-expr = { path = "../expr", version = "9.0.0" }
hashbrown = "0.12"