You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ul...@apache.org on 2021/10/12 02:50:21 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #1214] Refine sql
extension and avoid setup data for unused test
This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 6160b9d [KYUUBI #1214] Refine sql extension and avoid setup data for unused test
6160b9d is described below
commit 6160b9de30335ed74c60da0fe7bf3196acbcef57
Author: ulysses-you <ul...@gmail.com>
AuthorDate: Tue Oct 12 10:50:03 2021 +0800
[KYUUBI #1214] Refine sql extension and avoid setup data for unused test
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
- Refine rule order
- Speed up test
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #1214 from ulysses-you/tests.
Closes #1214
261271f0 [ulysses-you] test
Authored-by: ulysses-you <ul...@gmail.com>
Signed-off-by: ulysses-you <ul...@apache.org>
---
.../main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala | 5 +++--
.../scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala | 5 +++++
.../org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala | 5 +++++
.../scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala | 3 +--
.../src/test/scala/org/apache/spark/sql/WatchDogSuite.scala | 4 ++++
5 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/dev/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala b/dev/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala
index 7e4c780..e20052a 100644
--- a/dev/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala
+++ b/dev/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala
@@ -38,13 +38,14 @@ class KyuubiSparkSQLExtension extends (SparkSessionExtensions => Unit) {
extensions.injectParser{ case (_, parser) => new ZorderSparkSqlExtensionsParser(parser) }
extensions.injectResolutionRule(ResolveZorder)
+ // a help rule for ForcedMaxOutputRowsRule
+ extensions.injectResolutionRule(MarkAggregateOrderRule)
+
// Note that:
// InsertZorderBeforeWritingDatasource and InsertZorderBeforeWritingHive
// should be applied before
// RepartitionBeforeWrite and RepartitionBeforeWriteHive
// because we can only apply one of them (i.e. Global Sort or Repartition)
- extensions.injectResolutionRule(MarkAggregateOrderRule)
-
extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingDatasource)
extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingHive)
extensions.injectPostHocResolutionRule(KyuubiSqlClassification)
diff --git a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala
index 992105a..8592f4d 100644
--- a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala
+++ b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala
@@ -23,6 +23,11 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.kyuubi.sql.{FinalStageConfigIsolation, KyuubiSQLConf}
class FinalStageConfigIsolationSuite extends KyuubiSparkSQLExtensionTest {
+ protected override def beforeAll(): Unit = {
+ super.beforeAll()
+ setupData()
+ }
+
test("final stage config set reset check") {
withSQLConf(KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true",
"spark.sql.finalStage.adaptive.coalescePartitions.minPartitionNum" -> "1",
diff --git a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala
index 939fea8..7abeee4 100644
--- a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala
+++ b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala
@@ -23,6 +23,11 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.kyuubi.sql.KyuubiSQLConf
class InsertShuffleNodeBeforeJoinSuite extends KyuubiSparkSQLExtensionTest {
+ protected override def beforeAll(): Unit = {
+ super.beforeAll()
+ setupData()
+ }
+
test("force shuffle before join") {
def checkShuffleNodeNum(sqlString: String, num: Int): Unit = {
var expectedResult: Seq[Row] = Seq.empty
diff --git a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala
index 8ee0176..34259db 100644
--- a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala
+++ b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala
@@ -42,7 +42,6 @@ trait KyuubiSparkSQLExtensionTest extends QueryTest
.enableHiveSupport()
.getOrCreate())
}
- setupData()
super.beforeAll()
}
@@ -52,7 +51,7 @@ trait KyuubiSparkSQLExtensionTest extends QueryTest
_spark.foreach(_.stop)
}
- private def setupData(): Unit = {
+ protected def setupData(): Unit = {
val self = spark
import self.implicits._
spark.sparkContext.parallelize(
diff --git a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala
index 656e5bc..29ddea6 100644
--- a/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala
+++ b/dev/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala
@@ -23,6 +23,10 @@ import org.apache.kyuubi.sql.KyuubiSQLConf
import org.apache.kyuubi.sql.watchdog.MaxHivePartitionExceedException
class WatchDogSuite extends KyuubiSparkSQLExtensionTest {
+ protected override def beforeAll(): Unit = {
+ super.beforeAll()
+ setupData()
+ }
case class LimitAndExpected(limit: Int, expected: Int)
val limitAndExpecteds = List(LimitAndExpected(1, 1), LimitAndExpected(11, 10))