You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2020/04/19 23:26:54 UTC
[spark] branch branch-2.4 updated: [SPARK-31234][SQL][2.4]
ResetCommand should not affect static SQL Configuration
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 1af3b48 [SPARK-31234][SQL][2.4] ResetCommand should not affect static SQL Configuration
1af3b48 is described below
commit 1af3b48486e4b60f226ed44dcb23a7e6379c99a6
Author: gatorsmile <ga...@gmail.com>
AuthorDate: Sun Apr 19 16:24:56 2020 -0700
[SPARK-31234][SQL][2.4] ResetCommand should not affect static SQL Configuration
### What changes were proposed in this pull request?
This PR is to backport the fix of https://github.com/apache/spark/pull/28003, add a migration guide, update the PR description and add an end-to-end test case.
Before this PR, the SQL `RESET` command will reset the values of static SQL configuration to the default and remove the cached values of Spark Context Configurations in the current session. This PR fixes the bugs. After this PR, the `RESET` command follows its definition and only updates the runtime SQL configuration values to the default.
### Why are the changes needed?
When we introduced the feature of Static SQL Configuration, we did not update the implementation of SQL `RESET` command.
The static SQL configuration should not be changed by any command at runtime. However, the `RESET` command resets the values to the default. We should fix them.
### Does this PR introduce any user-facing change?
Before Spark 2.4.6, the `RESET` command resets both the runtime and static SQL configuration values to the default. It also removes the cached values of Spark Context Configurations in the current session, although these configuration values are for displaying/querying only.
### How was this patch tested?
Added an end-to-end test and a unit test
Closes #28262 from gatorsmile/spark-31234followup2.4.
Authored-by: gatorsmile <ga...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
docs/sql-migration-guide-upgrade.md | 6 +++++-
.../apache/spark/sql/execution/command/SetCommand.scala | 6 +++++-
.../org/apache/spark/sql/SparkSessionBuilderSuite.scala | 16 ++++++++++++++++
.../org/apache/spark/sql/internal/SQLConfSuite.scala | 15 +++++++++++++++
4 files changed, 41 insertions(+), 2 deletions(-)
diff --git a/docs/sql-migration-guide-upgrade.md b/docs/sql-migration-guide-upgrade.md
index 8170637..09cf834 100644
--- a/docs/sql-migration-guide-upgrade.md
+++ b/docs/sql-migration-guide-upgrade.md
@@ -7,7 +7,11 @@ displayTitle: Spark SQL Upgrading Guide
* Table of contents
{:toc}
-## Upgrading from Spark SQL 2.4 to 2.4.5
+## Upgrading from Spark SQL 2.4.5 to 2.4.6
+
+ - In Spark 2.4.6, the `RESET` command does not reset the static SQL configuration values to the default. It only clears the runtime SQL configuration values.
+
+## Upgrading from Spark SQL 2.4.4 to 2.4.5
- Starting from 2.4.5, SQL configurations are effective also when a Dataset is converted to an RDD and its
plan is executed due to action on the derived RDD. The previous behavior can be restored setting
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
index 3c900be..435cdf2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
@@ -164,7 +164,11 @@ object SetCommand {
case object ResetCommand extends RunnableCommand with Logging {
override def run(sparkSession: SparkSession): Seq[Row] = {
- sparkSession.sessionState.conf.clear()
+ val conf = sparkSession.sessionState.conf
+ conf.clear()
+ sparkSession.sparkContext.conf.getAll.foreach { case (k, v) =>
+ conf.setConfString(k, v)
+ }
Seq.empty[Row]
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index 44bf862..a57f09e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -21,6 +21,7 @@ import org.scalatest.BeforeAndAfterEach
import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
/**
* Test cases for the builder pattern of [[SparkSession]].
@@ -151,4 +152,19 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
session.sparkContext.hadoopConfiguration.unset(mySpecialKey)
}
}
+
+ test("SPARK-31234: RESET command will not change static sql configs and " +
+ "spark context conf values in SessionState") {
+ val session = SparkSession.builder()
+ .master("local")
+ .config(GLOBAL_TEMP_DATABASE.key, value = "globalTempDB-SPARK-31234")
+ .config("spark.app.name", "test-app-SPARK-31234")
+ .getOrCreate()
+
+ assert(session.sessionState.conf.getConfString("spark.app.name") === "test-app-SPARK-31234")
+ assert(session.sessionState.conf.getConf(GLOBAL_TEMP_DATABASE) === "globalTempDB-SPARK-31234")
+ session.sql("RESET")
+ assert(session.sessionState.conf.getConfString("spark.app.name") === "test-app-SPARK-31234")
+ assert(session.sessionState.conf.getConf(GLOBAL_TEMP_DATABASE) === "globalTempDB-SPARK-31234")
+ }
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index c9a6975..e965ad4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -114,6 +114,21 @@ class SQLConfSuite extends QueryTest with SharedSQLContext {
}
}
+ test("SPARK-31234: reset will not change static sql configs and spark core configs") {
+ val conf = spark.sparkContext.getConf.getAll.toMap
+ val appName = conf.get("spark.app.name")
+ val driverHost = conf.get("spark.driver.host")
+ val master = conf.get("spark.master")
+ val warehouseDir = conf.get("spark.sql.warehouse.dir")
+ // ensure the conf here is not default value, and will not be reset to default value later
+ assert(warehouseDir.get.contains(this.getClass.getCanonicalName))
+ sql("RESET")
+ assert(conf.get("spark.app.name") === appName)
+ assert(conf.get("spark.driver.host") === driverHost)
+ assert(conf.get("spark.master") === master)
+ assert(conf.get("spark.sql.warehouse.dir") === warehouseDir)
+ }
+
test("reset - public conf") {
spark.sessionState.conf.clear()
val original = spark.conf.get(SQLConf.GROUP_BY_ORDINAL)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org