You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2024/08/06 21:42:38 UTC
(datafusion-comet) branch main updated: chore: bump DataFusion to rev f4e519f (#783)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 6b64dcde chore: bump DataFusion to rev f4e519f (#783)
6b64dcde is described below
commit 6b64dcde40c3f28df3f02909ce15440fccaf2fca
Author: Huaxin Gao <hu...@apple.com>
AuthorDate: Tue Aug 6 14:42:34 2024 -0700
chore: bump DataFusion to rev f4e519f (#783)
* chore: bump DataFusion to rev c6f0d3c
* fix style
* Trigger Build
* Update native/Cargo.toml
---------
Co-authored-by: Andy Grove <ag...@apache.org>
---
native/Cargo.lock | 28 +++++++++----------
native/Cargo.toml | 14 +++++-----
native/core/src/execution/datafusion/planner.rs | 36 +++++++++++++++++++++----
3 files changed, 52 insertions(+), 26 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 3f6b1d1c..ef3d3cf2 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -804,7 +804,7 @@ dependencies = [
[[package]]
name = "datafusion"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"ahash",
"arrow",
@@ -851,7 +851,7 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"arrow-schema",
"async-trait",
@@ -948,7 +948,7 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"ahash",
"arrow",
@@ -968,7 +968,7 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"tokio",
]
@@ -976,7 +976,7 @@ dependencies = [
[[package]]
name = "datafusion-execution"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"arrow",
"chrono",
@@ -996,7 +996,7 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"ahash",
"arrow",
@@ -1014,7 +1014,7 @@ dependencies = [
[[package]]
name = "datafusion-functions"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"arrow",
"arrow-buffer",
@@ -1040,7 +1040,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"ahash",
"arrow",
@@ -1057,7 +1057,7 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"arrow",
"async-trait",
@@ -1076,7 +1076,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"ahash",
"arrow",
@@ -1105,7 +1105,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"ahash",
"arrow",
@@ -1118,7 +1118,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"datafusion-common",
"datafusion-execution",
@@ -1129,7 +1129,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"ahash",
"arrow",
@@ -1162,7 +1162,7 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "40.0.0"
-source = "git+https://github.com/apache/datafusion.git?rev=35c2e7e#35c2e7e7eb04e80877bbbc1fa4a5b06f31a4e4bc"
+source = "git+https://github.com/apache/datafusion.git?rev=c6f0d3c#c6f0d3cac93ef1436313160f1dba878745d693bb"
dependencies = [
"arrow",
"arrow-array",
diff --git a/native/Cargo.toml b/native/Cargo.toml
index c6cf571a..a9b7d4fb 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -39,13 +39,13 @@ arrow-buffer = { version = "52.2.0" }
arrow-data = { version = "52.2.0" }
arrow-schema = { version = "52.2.0" }
parquet = { version = "52.2.0", default-features = false, features = ["experimental"] }
-datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "35c2e7e" }
-datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "35c2e7e", features = ["unicode_expressions", "crypto_expressions"] }
-datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "35c2e7e", features = ["crypto_expressions"] }
-datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "35c2e7e", default-features = false }
-datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "35c2e7e", default-features = false }
-datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion.git", rev = "35c2e7e", default-features = false }
-datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "35c2e7e", default-features = false }
+datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "f4e519f" }
+datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "f4e519f", features = ["unicode_expressions", "crypto_expressions"] }
+datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "f4e519f", features = ["crypto_expressions"] }
+datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "f4e519f", default-features = false }
+datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "f4e519f", default-features = false }
+datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion.git", rev = "f4e519f", default-features = false }
+datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "f4e519f", default-features = false }
datafusion-comet-spark-expr = { path = "spark-expr", version = "0.2.0" }
datafusion-comet-proto = { path = "proto", version = "0.2.0" }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
diff --git a/native/core/src/execution/datafusion/planner.rs b/native/core/src/execution/datafusion/planner.rs
index 3cdf799c..836503aa 100644
--- a/native/core/src/execution/datafusion/planner.rs
+++ b/native/core/src/execution/datafusion/planner.rs
@@ -50,6 +50,8 @@ use crate::{
};
use arrow_schema::{DataType, Field, Schema, TimeUnit, DECIMAL128_MAX_PRECISION};
use datafusion::functions_aggregate::bit_and_or_xor::{bit_and_udaf, bit_or_udaf, bit_xor_udaf};
+use datafusion::functions_aggregate::min_max::max_udaf;
+use datafusion::functions_aggregate::min_max::min_udaf;
use datafusion::functions_aggregate::sum::sum_udaf;
use datafusion::physical_plan::windows::BoundedWindowAggExec;
use datafusion::physical_plan::InputOrderMode;
@@ -63,7 +65,7 @@ use datafusion::{
execution_props::ExecutionProps,
expressions::{
in_list, BinaryExpr, CaseExpr, CastExpr, Column, IsNotNullExpr, IsNullExpr,
- Literal as DataFusionLiteral, Max, Min, NotExpr,
+ Literal as DataFusionLiteral, NotExpr,
},
AggregateExpr, PhysicalExpr, PhysicalSortExpr, ScalarFunctionExpr,
},
@@ -1274,14 +1276,38 @@ impl PhysicalPlanner {
.map_err(|e| ExecutionError::DataFusionError(e.to_string()))
}
AggExprStruct::Min(expr) => {
- let child = self.create_expr(expr.child.as_ref().unwrap(), schema)?;
+ let child = self.create_expr(expr.child.as_ref().unwrap(), schema.clone())?;
let datatype = to_arrow_datatype(expr.datatype.as_ref().unwrap());
- Ok(Arc::new(Min::new(child, "min", datatype)))
+ let child = Arc::new(CastExpr::new(child, datatype.clone(), None));
+ create_aggregate_expr(
+ &min_udaf(),
+ &[child],
+ &[],
+ &[],
+ &[],
+ schema.as_ref(),
+ "min",
+ false,
+ false,
+ )
+ .map_err(|e| ExecutionError::DataFusionError(e.to_string()))
}
AggExprStruct::Max(expr) => {
- let child = self.create_expr(expr.child.as_ref().unwrap(), schema)?;
+ let child = self.create_expr(expr.child.as_ref().unwrap(), schema.clone())?;
let datatype = to_arrow_datatype(expr.datatype.as_ref().unwrap());
- Ok(Arc::new(Max::new(child, "max", datatype)))
+ let child = Arc::new(CastExpr::new(child, datatype.clone(), None));
+ create_aggregate_expr(
+ &max_udaf(),
+ &[child],
+ &[],
+ &[],
+ &[],
+ schema.as_ref(),
+ "max",
+ false,
+ false,
+ )
+ .map_err(|e| ExecutionError::DataFusionError(e.to_string()))
}
AggExprStruct::Sum(expr) => {
let child = self.create_expr(expr.child.as_ref().unwrap(), schema.clone())?;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org