You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ta...@apache.org on 2024/10/14 01:43:51 UTC
(incubator-gluten) branch main updated: fix ut (#7485)
This is an automated email from the ASF dual-hosted git repository.
taiyangli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cea377d11a fix ut (#7485)
cea377d11a is described below
commit cea377d11a986ba0426784165edf33fec5816fcb
Author: kevinyhzou <37...@users.noreply.github.com>
AuthorDate: Mon Oct 14 09:43:47 2024 +0800
fix ut (#7485)
---
cpp-ch/local-engine/Parser/scalar_function_parser/elementAt.cpp | 7 +++++++
.../apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 --
.../apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 --
.../apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 --
.../apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala | 2 --
5 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/elementAt.cpp b/cpp-ch/local-engine/Parser/scalar_function_parser/elementAt.cpp
index 021e9245e2..a3a2549e2f 100644
--- a/cpp-ch/local-engine/Parser/scalar_function_parser/elementAt.cpp
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/elementAt.cpp
@@ -58,6 +58,13 @@ public:
if (isMap(arg_type))
{
const auto * map_arg = parsed_args[0];
+ const DataTypeMap * map_type = checkAndGetDataType<DataTypeMap>(removeNullable(map_arg->result_type).get());
+ if (isNothing(map_type->getKeyType()))
+ {
+ const DataTypePtr value_type = map_type->getValueType();
+ const auto * null_const_node = addColumnToActionsDAG(actions_dag, makeNullable(value_type), Field{});
+ return null_const_node;
+ }
const auto * key_arg = parsed_args[1];
const auto * result_node = toFunctionNode(actions_dag, "arrayElementOrNull", {map_arg, key_arg});
return convertNodeTypeIfNeeded(substrait_func, result_node, actions_dag);
diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 0091c0c4dd..b8171bc633 100644
--- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -159,7 +159,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenDataFrameComplexTypeSuite]
enableSuite[GlutenDataFrameFunctionsSuite]
.exclude("map with arrays")
- .exclude("element_at function")
.exclude("flatten function")
.exclude("aggregate function - array for primitive type not containing null")
.exclude("aggregate function - array for primitive type containing null")
@@ -655,7 +654,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenCollectionExpressionsSuite]
.exclude("ArraysZip") // wait for https://github.com/ClickHouse/ClickHouse/pull/69576
.exclude("Sequence of numbers")
- .exclude("elementAt")
.exclude("Shuffle")
.exclude("SPARK-33386: element_at ArrayIndexOutOfBoundsException")
.exclude("SPARK-33460: element_at NoSuchElementException")
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 77be9e78c0..731f66c530 100644
--- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -177,7 +177,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenDataFrameComplexTypeSuite]
enableSuite[GlutenDataFrameFunctionsSuite]
.exclude("map with arrays")
- .exclude("element_at function")
.exclude("flatten function")
.exclude("aggregate function - array for primitive type not containing null")
.exclude("aggregate function - array for primitive type containing null")
@@ -683,7 +682,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenCollectionExpressionsSuite]
.exclude("ArraysZip") // wait for https://github.com/ClickHouse/ClickHouse/pull/69576
.exclude("Sequence of numbers")
- .exclude("elementAt")
.exclude("Shuffle")
.exclude("SPARK-33386: element_at ArrayIndexOutOfBoundsException")
.exclude("SPARK-33460: element_at NoSuchElementException")
diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index d73e44fdc0..86d235dc88 100644
--- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -179,7 +179,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenDataFrameComplexTypeSuite]
enableSuite[GlutenDataFrameFunctionsSuite]
.exclude("map with arrays")
- .exclude("element_at function")
.exclude("flatten function")
.exclude("aggregate function - array for primitive type not containing null")
.exclude("aggregate function - array for primitive type containing null")
@@ -572,7 +571,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenCollectionExpressionsSuite]
.exclude("ArraysZip") // wait for https://github.com/ClickHouse/ClickHouse/pull/69576
.exclude("Sequence of numbers")
- .exclude("elementAt")
.exclude("Shuffle")
.exclude("SPARK-33386: element_at ArrayIndexOutOfBoundsException")
.exclude("SPARK-33460: element_at NoSuchElementException")
diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 2bd1bacb4f..3fe2764c74 100644
--- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -179,7 +179,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenDataFrameComplexTypeSuite]
enableSuite[GlutenDataFrameFunctionsSuite]
.exclude("map with arrays")
- .exclude("element_at function")
.exclude("flatten function")
.exclude("aggregate function - array for primitive type not containing null")
.exclude("aggregate function - array for primitive type containing null")
@@ -572,7 +571,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenCollectionExpressionsSuite]
.exclude("ArraysZip") // wait for https://github.com/ClickHouse/ClickHouse/pull/69576
.exclude("Sequence of numbers")
- .exclude("elementAt")
.exclude("Shuffle")
.exclude("SPARK-33386: element_at ArrayIndexOutOfBoundsException")
.exclude("SPARK-33460: element_at NoSuchElementException")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org