You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2024/11/11 19:56:22 UTC
(datafusion-comet) 01/01: upmerge
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch comet-parquet-exec
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 16033d9bb06706eaff125bbff39f23891decb213
Merge: e52fe774 82a9b954
Author: Andy Grove <ag...@apache.org>
AuthorDate: Mon Nov 11 12:56:11 2024 -0700
upmerge
.../org/apache/comet/parquet/ColumnReader.java | 83 ++-
.../apache/comet/parquet/MetadataColumnReader.java | 20 +-
.../main/java/org/apache/comet/parquet/Native.java | 6 +-
.../main/scala/org/apache/comet/CometConf.scala | 77 +-
docs/source/user-guide/compatibility.md | 1 +
docs/source/user-guide/configs.md | 32 +-
docs/source/user-guide/kubernetes.md | 8 +-
native/Cargo.lock | 784 ++++++++++++++-------
native/Cargo.toml | 30 +-
native/core/benches/aggregate.rs | 2 +-
native/core/benches/bloom_filter_agg.rs | 2 +-
.../datafusion/expressions/sum_decimal.rs | 2 +-
native/core/src/execution/datafusion/planner.rs | 8 +-
native/core/src/parquet/mod.rs | 25 +-
native/proto/src/proto/expr.proto | 205 +-----
native/spark-expr/src/cast.rs | 97 ++-
.../src/scalar_funcs/hash_expressions.rs | 8 +-
.../apache/comet/CometSparkSessionExtensions.scala | 10 +-
.../org/apache/comet/expressions/CometCast.scala | 26 +-
.../org/apache/comet/serde/QueryPlanSerde.scala | 694 ++++++------------
.../scala/org/apache/comet/CometCastSuite.scala | 35 +-
.../org/apache/comet/exec/CometJoinSuite.scala | 3 +-
22 files changed, 1080 insertions(+), 1078 deletions(-)
diff --cc native/Cargo.lock
index 95595848,3649128f..c3a664ff
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@@ -57,9 -57,9 +57,9 @@@ dependencies =
[[package]]
name = "allocator-api2"
--version = "0.2.18"
++version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
++checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9"
[[package]]
name = "android-tzdata"
@@@ -436,18 -436,7 +436,18 @@@ checksum = "d640d25bc63c50fb1f0b545ffd8
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
- "brotli-decompressor",
+ "brotli-decompressor 2.5.1",
+]
+
+[[package]]
+name = "brotli"
- version = "6.0.0"
++version = "7.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
++checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor 4.0.1",
]
[[package]]
@@@ -694,9 -673,9 +694,9 @@@ dependencies =
[[package]]
name = "cpufeatures"
--version = "0.2.14"
++version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0"
++checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6"
dependencies = [
"libc",
]
@@@ -791,9 -770,9 +791,9 @@@ dependencies =
[[package]]
name = "csv"
--version = "1.3.0"
++version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
--checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
++checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf"
dependencies = [
"csv-core",
"itoa",
@@@ -2195,38 -2344,21 +2369,38 @@@ dependencies =
[[package]]
name = "parquet"
- version = "53.1.0"
+ version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "310c46a70a3ba90d98fec39fa2da6d9d731e544191da6fb56c9d199484d0dd3e"
+ checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e"
dependencies = [
"ahash",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-ipc",
+ "arrow-schema",
+ "arrow-select",
+ "base64",
- "brotli 6.0.0",
++ "brotli 7.0.0",
"bytes",
"chrono",
+ "flate2",
+ "futures",
"half",
- "hashbrown",
+ "hashbrown 0.14.5",
+ "lz4_flex",
"num",
"num-bigint",
+ "object_store",
"paste",
"seq-macro",
+ "snap",
"thrift",
+ "tokio",
"twox-hash 1.6.3",
+ "zstd 0.13.2",
+ "zstd-sys",
]
[[package]]
@@@ -2550,9 -2682,9 +2724,9 @@@ dependencies =
[[package]]
name = "regex-automata"
- version = "0.4.7"
-version = "0.4.8"
++version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
-checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
++checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
"aho-corasick",
"memchr",
@@@ -2591,9 -2723,9 +2765,9 @@@ dependencies =
[[package]]
name = "rustix"
- version = "0.38.37"
-version = "0.38.39"
++version = "0.38.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
-checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee"
++checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0"
dependencies = [
"bitflags 2.6.0",
"errno",
@@@ -2894,18 -3037,18 +3079,18 @@@ dependencies =
[[package]]
name = "thiserror"
- version = "1.0.63"
-version = "1.0.68"
++version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
-checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892"
++checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
- version = "1.0.63"
-version = "1.0.68"
++version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
-checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e"
++checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
diff --cc native/Cargo.toml
index f9af172e,8f6f515a..4b89231c
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@@ -30,23 -30,23 +30,23 @@@ license = "Apache-2.0
edition = "2021"
# Comet uses the same minimum Rust version as DataFusion
- rust-version = "1.75"
+ rust-version = "1.79"
[workspace.dependencies]
- arrow = { version = "53.1.0", features = ["prettyprint", "ffi", "chrono-tz"] }
- arrow-array = { version = "53.1.0" }
- arrow-buffer = { version = "53.1.0" }
- arrow-data = { version = "53.1.0" }
- arrow-schema = { version = "53.1.0" }
- parquet = { version = "53.1.0", default-features = false, features = ["experimental"] }
- datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2" }
- datafusion = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false, features = ["unicode_expressions", "crypto_expressions", "parquet"] }
- datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", features = ["crypto_expressions"] }
- datafusion-functions-nested = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false }
- datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false }
- datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false }
- datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false }
- datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "3b6aac2", default-features = false }
+ arrow = { version = "53.2.0", features = ["prettyprint", "ffi", "chrono-tz"] }
+ arrow-array = { version = "53.2.0" }
+ arrow-buffer = { version = "53.2.0" }
+ arrow-data = { version = "53.2.0" }
+ arrow-schema = { version = "53.2.0" }
+ parquet = { version = "53.2.0", default-features = false, features = ["experimental"] }
+ datafusion-common = { version = "43.0.0" }
-datafusion = { version = "43.0.0", default-features = false, features = ["unicode_expressions", "crypto_expressions"] }
++datafusion = { version = "43.0.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "parquet"] }
+ datafusion-functions = { version = "43.0.0", features = ["crypto_expressions"] }
+ datafusion-functions-nested = { version = "43.0.0", default-features = false }
+ datafusion-expr = { version = "43.0.0", default-features = false }
+ datafusion-execution = { version = "43.0.0", default-features = false }
+ datafusion-physical-plan = { version = "43.0.0", default-features = false }
+ datafusion-physical-expr = { version = "43.0.0", default-features = false }
datafusion-comet-spark-expr = { path = "spark-expr", version = "0.4.0" }
datafusion-comet-proto = { path = "proto", version = "0.4.0" }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org