You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2021/07/15 08:35:33 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #807] Make
kyuubi-extension-spark_3.1 test pass with Spark 3.2.0-snapshot
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new ca94d7b [KYUUBI #807] Make kyuubi-extension-spark_3.1 test pass with Spark 3.2.0-snapshot
ca94d7b is described below
commit ca94d7b46c618c74c7da990bd09268ee94ddc99f
Author: ulysses-you <ul...@gmail.com>
AuthorDate: Thu Jul 15 16:35:21 2021 +0800
[KYUUBI #807] Make kyuubi-extension-spark_3.1 test pass with Spark 3.2.0-snapshot
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
Make this extesion more robust.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request
Closes #807 from ulysses-you/sql-extension.
Closes #807
27155c1e [ulysses-you] fix
Authored-by: ulysses-you <ul...@gmail.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../test/scala/org/apache/spark/sql/KyuubiExtensionSuite.scala | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/dev/kyuubi-extension-spark_3.1/src/test/scala/org/apache/spark/sql/KyuubiExtensionSuite.scala b/dev/kyuubi-extension-spark_3.1/src/test/scala/org/apache/spark/sql/KyuubiExtensionSuite.scala
index 9afe6c4..af42d8c 100644
--- a/dev/kyuubi-extension-spark_3.1/src/test/scala/org/apache/spark/sql/KyuubiExtensionSuite.scala
+++ b/dev/kyuubi-extension-spark_3.1/src/test/scala/org/apache/spark/sql/KyuubiExtensionSuite.scala
@@ -39,6 +39,7 @@ class KyuubiExtensionSuite extends QueryTest with SQLTestUtils with AdaptiveSpar
"org.apache.kyuubi.sql.KyuubiSparkSQLExtension")
.config(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")
.config("spark.hadoop.hive.exec.dynamic.partition.mode", "nonstrict")
+ .config("spark.hadoop.hive.metastore.client.capability.check", "false")
.config("spark.ui.enabled", "false")
.enableHiveSupport()
.getOrCreate()
@@ -328,6 +329,7 @@ class KyuubiExtensionSuite extends QueryTest with SQLTestUtils with AdaptiveSpar
SQLConf.SHUFFLE_PARTITIONS.key -> "3",
KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true",
"spark.sql.adaptive.advisoryPartitionSizeInBytes" -> "1",
+ "spark.sql.adaptive.coalescePartitions.minPartitionSize" -> "1",
"spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes" -> "10000000") {
// use loop to double check final stage config doesn't affect the sql query each other
@@ -372,10 +374,10 @@ class KyuubiExtensionSuite extends QueryTest with SQLTestUtils with AdaptiveSpar
// test ReusedExchange
checkPartitionNum(
"""
- |SELECT t0.c2 FROM (
- |SELECT t1.c1, count(*) as c2 FROM t1 GROUP BY t1.c1
+ |SELECT /*+ REPARTITION */ t0.c2 FROM (
+ |SELECT t1.c1, (count(*) + c1) as c2 FROM t1 GROUP BY t1.c1
|) t0 JOIN (
- |SELECT t1.c1, count(*) as c2 FROM t1 GROUP BY t1.c1
+ |SELECT t1.c1, (count(*) + c1) as c2 FROM t1 GROUP BY t1.c1
|) t1 ON t0.c2 = t1.c2
|""".stripMargin,
3,