You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ph...@apache.org on 2024/05/11 00:05:20 UTC
(incubator-gluten) branch main updated: [GLUTEN-4039][VL] Add flatten function support (#5551)
This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d93100689 [GLUTEN-4039][VL] Add flatten function support (#5551)
d93100689 is described below
commit d93100689a152fee685fde4f291de7fc7c400290
Author: Tengfei Huang <te...@gmail.com>
AuthorDate: Sat May 11 08:05:15 2024 +0800
[GLUTEN-4039][VL] Add flatten function support (#5551)
---
.../org/apache/gluten/utils/CHExpressionUtil.scala | 3 ++-
.../execution/ScalarFunctionsValidateSuite.scala | 20 ++++++++++++++++++++
docs/velox-backend-support-progress.md | 2 +-
.../gluten/expression/ExpressionMappings.scala | 1 +
.../apache/gluten/expression/ExpressionNames.scala | 1 +
5 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
index b4190f1b8..f593e5fac 100644
--- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
@@ -198,6 +198,7 @@ object CHExpressionUtil {
UNIX_MILLIS -> DefaultValidator(),
UNIX_MICROS -> DefaultValidator(),
TIMESTAMP_MILLIS -> DefaultValidator(),
- TIMESTAMP_MICROS -> DefaultValidator()
+ TIMESTAMP_MICROS -> DefaultValidator(),
+ FLATTEN -> DefaultValidator()
)
}
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 233a1ca96..485d70f9d 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -941,4 +941,24 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest {
}
}
+ test("test flatten nested array") {
+ withTempPath {
+ path =>
+ Seq[Seq[Seq[Integer]]](
+ Seq(Seq(1, 2), Seq(4, 5)),
+ null,
+ Seq(null, Seq(1, 2)),
+ Seq(null, null),
+ Seq(Seq(1, 2, null), Seq(null, null), Seq(3, 4), Seq.empty))
+ .toDF("arrays")
+ .write
+ .parquet(path.getCanonicalPath)
+
+ spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("array_tbl")
+
+ runQueryAndCompare("select flatten(arrays) as res from array_tbl;") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
}
diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md
index 1171b7d91..3d1d25be0 100644
--- a/docs/velox-backend-support-progress.md
+++ b/docs/velox-backend-support-progress.md
@@ -281,7 +281,7 @@ Gluten supports 199 functions. (Drag to right to see all data types)
| explode_outer, explode | | | | | | | | | | | | | | | | | | | | | | |
| filter | filter | filter | S | Lambda with index argument not supported | | | | | | | | | | | | | | | | | | |
| forall | all_match | | S | | | | | | | | | | | | | | | | | | | |
-| flatten | flatten | | | | | | | | | | | | | | | | | | | | | |
+| flatten | flatten | flatten | S | | | | | | | | | | | | | | | | | | | |
| map | map | map | S | | | | | | | | | | | | | | | | | | | |
| map_concat | map_concat | | | | | | | | | | | | | | | | | | | | | |
| map_entries | map_entries | | | | | | | | | | | | | | | | | | | | | |
diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
index ef43c2724..33e4f0a7b 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
@@ -241,6 +241,7 @@ object ExpressionMappings {
Sig[ArrayExists](EXISTS),
Sig[Shuffle](SHUFFLE),
Sig[ZipWith](ZIP_WITH),
+ Sig[Flatten](FLATTEN),
// Map functions
Sig[CreateMap](CREATE_MAP),
Sig[GetMapValue](GET_MAP_VALUE),
diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 54a55b355..b9e247a8d 100644
--- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -258,6 +258,7 @@ object ExpressionNames {
final val TRANSFORM = "transform"
final val SHUFFLE = "shuffle"
final val ZIP_WITH = "zip_with"
+ final val FLATTEN = "flatten"
// Map functions
final val CREATE_MAP = "map"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org