You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ru...@apache.org on 2025/03/12 16:39:25 UTC
(incubator-gluten) branch main updated: [VL] Enable map_concat function (#8781)
This is an automated email from the ASF dual-hosted git repository.
rui pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 30cbd88fc7 [VL] Enable map_concat function (#8781)
30cbd88fc7 is described below
commit 30cbd88fc71abd119a5a4e27e5119d750bd83fb5
Author: Rui Mo <ru...@apache.org>
AuthorDate: Wed Mar 12 16:39:21 2025 +0000
[VL] Enable map_concat function (#8781)
---
.../org/apache/gluten/utils/CHExpressionUtil.scala | 3 ++-
.../execution/ScalarFunctionsValidateSuite.scala | 18 ++++++++++++++++++
cpp/core/config/GlutenConfig.h | 1 +
cpp/velox/compute/WholeStageResultIterator.cc | 6 ++++++
.../apache/gluten/expression/ExpressionMappings.scala | 1 +
.../scala/org/apache/gluten/config/GlutenConfig.scala | 3 ++-
.../org/apache/gluten/expression/ExpressionNames.scala | 1 +
7 files changed, 31 insertions(+), 2 deletions(-)
diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
index 4a9ce71c79..6e3d2ec19a 100644
--- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala
@@ -204,6 +204,7 @@ object CHExpressionUtil {
STACK -> DefaultValidator(),
RAISE_ERROR -> DefaultValidator(),
WIDTH_BUCKET -> DefaultValidator(),
- MAKE_DATE -> DefaultValidator()
+ MAKE_DATE -> DefaultValidator(),
+ MAP_CONCAT -> DefaultValidator()
)
}
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 73d1ec68e6..3b5865a3a3 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -578,6 +578,24 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}
+ test("map_concat") {
+ withTempPath {
+ path =>
+ Seq(
+ Map[String, Int]("a" -> 1, "b" -> 2),
+ Map[String, Int]("a" -> 2, "b" -> 3),
+ null
+ )
+ .toDF("m")
+ .write
+ .parquet(path.getCanonicalPath)
+ spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("map_tbl")
+ runQueryAndCompare("select map_concat(m, map('c', 4)) from map_tbl") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
test("map_filter") {
withTempPath {
path =>
diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h
index 3207d3bec5..95bfd8751d 100644
--- a/cpp/core/config/GlutenConfig.h
+++ b/cpp/core/config/GlutenConfig.h
@@ -75,6 +75,7 @@ const std::string kSparkRedactionString = "*********(redacted)";
const std::string kSparkLegacyTimeParserPolicy = "spark.sql.legacy.timeParserPolicy";
const std::string kShuffleFileBufferSize = "spark.shuffle.file.buffer";
+const std::string kSparkMapKeyDedupPolicy = "spark.sql.mapKeyDedupPolicy";
std::unordered_map<std::string, std::string>
parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength);
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc
index 5715771166..bd51da7416 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -566,6 +566,12 @@ std::unordered_map<std::string, std::string> WholeStageResultIterator::getQueryC
configs[velox::core::QueryConfig::kSparkLegacyDateFormatter] = "false";
}
+ if (veloxCfg_->get<std::string>(kSparkMapKeyDedupPolicy, "") == "EXCEPTION") {
+ configs[velox::core::QueryConfig::kThrowExceptionOnDuplicateMapKeys] = "true";
+ } else {
+ configs[velox::core::QueryConfig::kThrowExceptionOnDuplicateMapKeys] = "false";
+ }
+
const auto setIfExists = [&](const std::string& glutenKey, const std::string& veloxKey) {
const auto valueOptional = veloxCfg_->get<std::string>(glutenKey);
if (valueOptional.hasValue()) {
diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
index dc41bd0629..b14401cb6e 100644
--- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
+++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
@@ -259,6 +259,7 @@ object ExpressionMappings {
// Map functions
Sig[CreateMap](CREATE_MAP),
Sig[GetMapValue](GET_MAP_VALUE),
+ Sig[MapConcat](MAP_CONCAT),
Sig[MapKeys](MAP_KEYS),
Sig[MapValues](MAP_VALUES),
Sig[MapFromArrays](MAP_FROM_ARRAYS),
diff --git a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index a511a55607..af77bc84d6 100644
--- a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -506,7 +506,8 @@ object GlutenConfig {
(
GLUTEN_COLUMNAR_TO_ROW_MEM_THRESHOLD.key,
GLUTEN_COLUMNAR_TO_ROW_MEM_THRESHOLD.defaultValue.get.toString),
- (SPARK_SHUFFLE_SPILL_COMPRESS, SPARK_SHUFFLE_SPILL_COMPRESS_DEFAULT.toString)
+ (SPARK_SHUFFLE_SPILL_COMPRESS, SPARK_SHUFFLE_SPILL_COMPRESS_DEFAULT.toString),
+ (SQLConf.MAP_KEY_DEDUP_POLICY.key, SQLConf.MAP_KEY_DEDUP_POLICY.defaultValueString)
)
keyWithDefault.forEach(e => nativeConfMap.put(e._1, conf.getOrElse(e._1, e._2)))
GlutenConfigUtil.mapByteConfValue(
diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index ed4de623ca..148933e142 100644
--- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -282,6 +282,7 @@ object ExpressionNames {
// Map functions
final val CREATE_MAP = "map"
final val GET_MAP_VALUE = "get_map_value"
+ final val MAP_CONCAT = "map_concat"
final val MAP_KEYS = "map_keys"
final val MAP_VALUES = "map_values"
final val MAP_FROM_ARRAYS = "map_from_arrays"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org