You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2022/10/03 23:24:28 UTC
[arrow-datafusion] branch master updated: Upgrade `arrow` `parquet` and `arrow-flight` to 24.0.0 (#3691)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 011e6f5d5 Upgrade `arrow` `parquet` and `arrow-flight` to 24.0.0 (#3691)
011e6f5d5 is described below
commit 011e6f5d544a97f4e835dd28a2f5a41570acad3d
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Mon Oct 3 19:24:22 2022 -0400
Upgrade `arrow` `parquet` and `arrow-flight` to 24.0.0 (#3691)
* Update to arrow 24.0.0
* Update pyo3 interface
* Update datafusion-cli lockfile
---
datafusion-cli/Cargo.lock | 104 ++++++++++++++++++++++------------
datafusion-cli/Cargo.toml | 2 +-
datafusion-examples/Cargo.toml | 2 +-
datafusion/common/Cargo.toml | 4 +-
datafusion/common/src/pyarrow.rs | 6 +-
datafusion/core/Cargo.toml | 6 +-
datafusion/core/fuzz-utils/Cargo.toml | 2 +-
datafusion/expr/Cargo.toml | 2 +-
datafusion/jit/Cargo.toml | 2 +-
datafusion/optimizer/Cargo.toml | 2 +-
datafusion/physical-expr/Cargo.toml | 2 +-
datafusion/proto/Cargo.toml | 2 +-
datafusion/row/Cargo.toml | 2 +-
datafusion/sql/Cargo.toml | 2 +-
14 files changed, 87 insertions(+), 53 deletions(-)
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index bdb0fdbae..bc4e084f1 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -79,12 +79,15 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "arrow"
-version = "23.0.0"
+version = "24.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fedc767fbaa36ea50f086215f54f1a007d22046fc4754b0448c657bcbe9f8413"
+checksum = "d68391300d5237f6725f0f869ae7cb65d45fcf8a6d18f6ceecd328fb803bef93"
dependencies = [
"ahash 0.8.0",
+ "arrow-array",
"arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
"bitflags",
"chrono",
"comfy-table",
@@ -99,20 +102,52 @@ dependencies = [
"num",
"regex",
"regex-syntax",
- "serde",
"serde_json",
]
+[[package]]
+name = "arrow-array"
+version = "24.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0bb00c5862b5eea683812083c495bef01a9a5149da46ad2f4c0e4aa8800f64d"
+dependencies = [
+ "ahash 0.8.0",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "chrono",
+ "half",
+ "hashbrown",
+ "num",
+]
+
[[package]]
name = "arrow-buffer"
-version = "23.0.0"
+version = "24.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d290050c6e12a81a24ad08525cef2203c4156a6350f75508d49885d677e88ea9"
+checksum = "3e594d0fe0026a8bc2459bdc5ac9623e5fb666724a715e0acbc96ba30c5d4cc7"
dependencies = [
"half",
+]
+
+[[package]]
+name = "arrow-data"
+version = "24.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8500df05060d86fdc53e9b5cb32e51bfeaacc040fdeced3eb99ac0d59200ff45"
+dependencies = [
+ "arrow-buffer",
+ "arrow-schema",
+ "half",
"num",
]
+[[package]]
+name = "arrow-schema"
+version = "24.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86d1fef01f25e1452c86fa6887f078de8e0aaeeb828370feab205944cfc30e27"
+
[[package]]
name = "async-trait"
version = "0.1.57"
@@ -438,7 +473,7 @@ dependencies = [
"log",
"num_cpus",
"object_store",
- "ordered-float 3.1.0",
+ "ordered-float 3.2.0",
"parking_lot",
"parquet",
"paste",
@@ -475,7 +510,7 @@ version = "12.0.0"
dependencies = [
"arrow",
"object_store",
- "ordered-float 3.1.0",
+ "ordered-float 3.2.0",
"parquet",
"sqlparser",
]
@@ -519,7 +554,7 @@ dependencies = [
"hashbrown",
"lazy_static",
"md-5",
- "ordered-float 3.1.0",
+ "ordered-float 3.2.0",
"paste",
"rand",
"regex",
@@ -974,9 +1009,9 @@ dependencies = [
[[package]]
name = "iana-time-zone"
-version = "0.1.49"
+version = "0.1.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bbaead50122b06e9a973ac20bc7445074d99ad9a0a0654934876908a9cec82c"
+checksum = "fd911b35d940d2bd0bea0f9100068e5b97b51a1cbe13d13382f132e0365257a0"
dependencies = [
"android_system_properties",
"core-foundation-sys",
@@ -1143,9 +1178,9 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.133"
+version = "0.2.134"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966"
+checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb"
[[package]]
name = "libm"
@@ -1430,9 +1465,9 @@ dependencies = [
[[package]]
name = "ordered-float"
-version = "3.1.0"
+version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98ffdb14730ed2ef599c65810c15b000896e21e8776b512de0db0c3d7335cc2a"
+checksum = "129d36517b53c461acc6e1580aeb919c8ae6708a4b1eae61c4463a615d4f0411"
dependencies = [
"num-traits",
]
@@ -1468,9 +1503,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "23.0.0"
+version = "24.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc38abcd826e52e80a45abe46707745e6d12d31031a9b11c2eab112def7efe71"
+checksum = "74fd590f0672998df84503d1bcbebc69732583d03cc3495c7dd8d3e5a1d8437f"
dependencies = [
"ahash 0.8.0",
"arrow",
@@ -1554,9 +1589,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
[[package]]
name = "proc-macro2"
-version = "1.0.43"
+version = "1.0.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
+checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b"
dependencies = [
"unicode-ident",
]
@@ -1729,9 +1764,9 @@ dependencies = [
[[package]]
name = "rustix"
-version = "0.35.10"
+version = "0.35.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af895b90e5c071badc3136fc10ff0bcfc98747eadbaf43ed8f214e07ba8f8477"
+checksum = "fbb2fda4666def1433b1b05431ab402e42a1084285477222b72d6c564c417cef"
dependencies = [
"bitflags",
"errno",
@@ -1893,9 +1928,9 @@ dependencies = [
[[package]]
name = "smallvec"
-version = "1.9.0"
+version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
+checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
[[package]]
name = "snafu"
@@ -1943,9 +1978,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "sqlparser"
-version = "0.23.0"
+version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0beb13adabbdda01b63d595f38c8bfd19a361e697fd94ce0098a634077bc5b25"
+checksum = "dac9c312566fdfc45a38ecf1924013c82af2a7d5315e46f67b1cc987f12be260"
dependencies = [
"log",
]
@@ -1995,9 +2030,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
-version = "1.0.100"
+version = "1.0.101"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e"
+checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2"
dependencies = [
"proc-macro2",
"quote",
@@ -2035,18 +2070,18 @@ checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16"
[[package]]
name = "thiserror"
-version = "1.0.35"
+version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85"
+checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
-version = "1.0.35"
+version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783"
+checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
dependencies = [
"proc-macro2",
"quote",
@@ -2090,9 +2125,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
-version = "1.21.1"
+version = "1.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95"
+checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099"
dependencies = [
"autocfg",
"bytes",
@@ -2100,7 +2135,6 @@ dependencies = [
"memchr",
"mio",
"num_cpus",
- "once_cell",
"parking_lot",
"pin-project-lite",
"socket2",
@@ -2391,9 +2425,9 @@ dependencies = [
[[package]]
name = "webpki-roots"
-version = "0.22.4"
+version = "0.22.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf"
+checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be"
dependencies = [
"webpki",
]
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index e74b81a36..84c4bab48 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.62"
readme = "README.md"
[dependencies]
-arrow = "23.0.0"
+arrow = "24.0.0"
clap = { version = "3", features = ["derive", "cargo"] }
datafusion = { path = "../datafusion/core", version = "12.0.0" }
dirs = "4.0.0"
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index c1feb1542..12c267832 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -34,7 +34,7 @@ path = "examples/avro_sql.rs"
required-features = ["datafusion/avro"]
[dev-dependencies]
-arrow-flight = "23.0.0"
+arrow-flight = "24.0.0"
async-trait = "0.1.41"
datafusion = { path = "../datafusion/core" }
futures = "0.3"
diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
index 48bc61f53..994c39bef 100644
--- a/datafusion/common/Cargo.toml
+++ b/datafusion/common/Cargo.toml
@@ -40,10 +40,10 @@ pyarrow = ["pyo3", "arrow/pyarrow"]
[dependencies]
apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true }
-arrow = { version = "23.0.0", default-features = false }
+arrow = { version = "24.0.0", default-features = false }
cranelift-module = { version = "0.88.0", optional = true }
object_store = { version = "0.5.0", default-features = false, optional = true }
ordered-float = "3.0"
-parquet = { version = "23.0.0", default-features = false, optional = true }
+parquet = { version = "24.0.0", default-features = false, optional = true }
pyo3 = { version = "0.17.1", optional = true }
sqlparser = "0.24"
diff --git a/datafusion/common/src/pyarrow.rs b/datafusion/common/src/pyarrow.rs
index bb3d9a6d7..6d9cab44c 100644
--- a/datafusion/common/src/pyarrow.rs
+++ b/datafusion/common/src/pyarrow.rs
@@ -43,8 +43,8 @@ impl PyArrowConvert for ScalarValue {
let array = factory.call1((args, typ))?;
// convert the pyarrow array to rust array using C data interface
- let array = array.extract::<ArrayData>()?;
- let scalar = ScalarValue::try_from_array(&array.into(), 0)?;
+ let array = arrow::array::make_array(ArrayData::from_pyarrow(array)?);
+ let scalar = ScalarValue::try_from_array(&array, 0)?;
Ok(scalar)
}
@@ -52,7 +52,7 @@ impl PyArrowConvert for ScalarValue {
fn to_pyarrow(&self, py: Python) -> PyResult<PyObject> {
let array = self.to_array();
// convert to pyarrow array using C data interface
- let pyarray = array.data_ref().clone().into_py(py);
+ let pyarray = array.data().to_pyarrow(py)?;
let pyscalar = pyarray.call_method1(py, "__getitem__", (0,))?;
Ok(pyscalar)
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index 1e338c5d1..8c275e4da 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -56,7 +56,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
[dependencies]
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
apache-avro = { version = "0.14", optional = true }
-arrow = { version = "23.0.0", features = ["prettyprint"] }
+arrow = { version = "24.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
bytes = "1.1"
chrono = { version = "0.4", default-features = false }
@@ -78,7 +78,7 @@ num_cpus = "1.13.0"
object_store = "0.5.0"
ordered-float = "3.0"
parking_lot = "0.12"
-parquet = { version = "23.0.0", features = ["arrow", "async"] }
+parquet = { version = "24.0.0", features = ["arrow", "async"] }
paste = "^1.0"
pin-project-lite = "^0.2.7"
pyo3 = { version = "0.17.1", optional = true }
@@ -93,7 +93,7 @@ url = "2.2"
uuid = { version = "1.0", features = ["v4"] }
[dev-dependencies]
-arrow = { version = "23.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
+arrow = { version = "24.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
async-trait = "0.1.53"
criterion = "0.4"
csv = "1.1.6"
diff --git a/datafusion/core/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml
index 07f09b636..3f1f2ed76 100644
--- a/datafusion/core/fuzz-utils/Cargo.toml
+++ b/datafusion/core/fuzz-utils/Cargo.toml
@@ -23,6 +23,6 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-arrow = { version = "23.0.0", features = ["prettyprint"] }
+arrow = { version = "24.0.0", features = ["prettyprint"] }
env_logger = "0.9.0"
rand = "0.8"
diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
index e0b8e2d52..a3111517e 100644
--- a/datafusion/expr/Cargo.toml
+++ b/datafusion/expr/Cargo.toml
@@ -36,6 +36,6 @@ path = "src/lib.rs"
[dependencies]
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "23.0.0", default-features = false }
+arrow = { version = "24.0.0", default-features = false }
datafusion-common = { path = "../common", version = "12.0.0" }
sqlparser = "0.24"
diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml
index 3ac81631a..6166e9174 100644
--- a/datafusion/jit/Cargo.toml
+++ b/datafusion/jit/Cargo.toml
@@ -36,7 +36,7 @@ path = "src/lib.rs"
jit = []
[dependencies]
-arrow = { version = "23.0.0", default-features = false }
+arrow = { version = "24.0.0", default-features = false }
cranelift = "0.88.0"
cranelift-jit = "0.88.0"
cranelift-module = "0.88.0"
diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml
index 409521088..862f51024 100644
--- a/datafusion/optimizer/Cargo.toml
+++ b/datafusion/optimizer/Cargo.toml
@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []
[dependencies]
-arrow = { version = "23.0.0", features = ["prettyprint"] }
+arrow = { version = "24.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
chrono = { version = "0.4", default-features = false }
datafusion-common = { path = "../common", version = "12.0.0" }
diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
index ba618acb6..a2473fb75 100644
--- a/datafusion/physical-expr/Cargo.toml
+++ b/datafusion/physical-expr/Cargo.toml
@@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"]
[dependencies]
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "23.0.0", features = ["prettyprint"] }
+arrow = { version = "24.0.0", features = ["prettyprint"] }
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
chrono = { version = "0.4", default-features = false }
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index adab4b356..122b980cf 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -40,7 +40,7 @@ default = []
json = ["pbjson", "pbjson-build", "serde", "serde_json"]
[dependencies]
-arrow = "23.0.0"
+arrow = "24.0.0"
datafusion = { path = "../core", version = "12.0.0" }
datafusion-common = { path = "../common", version = "12.0.0" }
datafusion-expr = { path = "../expr", version = "12.0.0" }
diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml
index d1c7fdf0c..306ab7b3c 100644
--- a/datafusion/row/Cargo.toml
+++ b/datafusion/row/Cargo.toml
@@ -37,7 +37,7 @@ path = "src/lib.rs"
jit = ["datafusion-jit"]
[dependencies]
-arrow = "23.0.0"
+arrow = "24.0.0"
datafusion-common = { path = "../common", version = "12.0.0" }
datafusion-jit = { path = "../jit", version = "12.0.0", optional = true }
paste = "^1.0"
diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml
index cf057a562..55aabd48c 100644
--- a/datafusion/sql/Cargo.toml
+++ b/datafusion/sql/Cargo.toml
@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []
[dependencies]
-arrow = { version = "23.0.0", default-features = false }
+arrow = { version = "24.0.0", default-features = false }
datafusion-common = { path = "../common", version = "12.0.0" }
datafusion-expr = { path = "../expr", version = "12.0.0" }
sqlparser = "0.24"