You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by yu...@apache.org on 2025/05/02 13:26:03 UTC
(incubator-gluten) branch main updated: [GLUTEN-9496][VL] Enable concat function with array datatype for spark (#9497)
This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 018eee6d37 [GLUTEN-9496][VL] Enable concat function with array datatype for spark (#9497)
018eee6d37 is described below
commit 018eee6d377b5cd0c876c37d100239215d1c083e
Author: dcoliversun <qi...@gmail.com>
AuthorDate: Fri May 2 21:25:58 2025 +0800
[GLUTEN-9496][VL] Enable concat function with array datatype for spark (#9497)
This PR aims to enable concat function with array datatype for spark.
(Fixes: #9496)
---
.../gluten/execution/ScalarFunctionsValidateSuite.scala | 17 +++++++++++++++++
cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc | 3 +--
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 38c1802720..564fcd45d6 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -1172,6 +1172,23 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}
+ test("test concat with array") {
+ withTempPath {
+ path =>
+ Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2), Seq.empty, null)
+ .toDF("value")
+ .write
+ .parquet(path.getCanonicalPath)
+
+ spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("array_tbl")
+
+ runQueryAndCompare("select concat(value, array(1)) from array_tbl;") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+
+ }
+ }
+
test("test array transform") {
withTable("t") {
sql("create table t (arr ARRAY<INT>) using parquet")
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index ecb69fff6c..67f2634cd4 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -211,8 +211,7 @@ bool SubstraitToVeloxPlanValidator::validateScalarFunction(
}
if (name == "concat") {
for (const auto& type : types) {
- if (type.find("struct") != std::string::npos || type.find("map") != std::string::npos ||
- type.find("list") != std::string::npos) {
+ if (type.find("struct") != std::string::npos || type.find("map") != std::string::npos) {
LOG_VALIDATION_MSG(type + " is not supported in concat.");
return false;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org