You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2020/02/20 16:10:04 UTC
[spark] branch branch-3.0 updated:
[SPARK-29930][SPARK-30416][SQL][FOLLOWUP] Move deprecated/removed config
checks from RuntimeConfig to SQLConf
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 68d8ff1 [SPARK-29930][SPARK-30416][SQL][FOLLOWUP] Move deprecated/removed config checks from RuntimeConfig to SQLConf
68d8ff1 is described below
commit 68d8ff1c5005122043084b94e0e44ecd6d91f4f0
Author: Maxim Gekk <ma...@gmail.com>
AuthorDate: Fri Feb 21 00:00:48 2020 +0800
[SPARK-29930][SPARK-30416][SQL][FOLLOWUP] Move deprecated/removed config checks from RuntimeConfig to SQLConf
### What changes were proposed in this pull request?
- Output warnings for deprecated SQL configs in `SQLConf. setConfWithCheck()` and in `SQLConf. unsetConf()`
- Throw an exception for removed SQL configs in `SQLConf. setConfWithCheck()` when they set to non-default values
- Remove checking of deprecated and removed SQL configs from RuntimeConfig
### Why are the changes needed?
Currently, warnings/exceptions are printed only when a SQL config is set dynamically, for instance via `spark.conf.set()`. After the changes, removed/deprecated SQL configs will be checked when they set statically. For example:
```
$ bin/spark-shell --conf spark.sql.fromJsonForceNullableSchema=false
scala> spark.emptyDataFrame
java.lang.IllegalArgumentException: Error while instantiating 'org.apache.spark.sql.hive.HiveSessionStateBuilder':
...
Caused by: org.apache.spark.sql.AnalysisException: The SQL config 'spark.sql.fromJsonForceNullableSchema' was removed in the version 3.0.0. It was removed to prevent errors like SPARK-23173 for non-default value.
```
```
$ bin/spark-shell --conf spark.sql.hive.verifyPartitionPath=false
scala> spark.emptyDataFrame
20/02/20 02:10:26 WARN SQLConf: The SQL config 'spark.sql.hive.verifyPartitionPath' has been deprecated in Spark v3.0 and may be removed in the future. This config is replaced by 'spark.files.ignoreMissingFiles'.
```
### Does this PR introduce any user-facing change?
Yes
### How was this patch tested?
By `SQLConfSuite`
Closes #27645 from MaxGekk/remove-sql-configs-followup-2.
Authored-by: Maxim Gekk <ma...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit a551715fd26779a0e02534d81d1c3f0804152fec)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../org/apache/spark/sql/internal/SQLConf.scala | 28 +++++++++++++++++++++-
.../scala/org/apache/spark/sql/RuntimeConfig.scala | 27 +--------------------
2 files changed, 28 insertions(+), 27 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 9cbaaee..2d72344 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -33,6 +33,7 @@ import org.apache.spark.internal.Logging
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.{IGNORE_MISSING_FILES => SPARK_IGNORE_MISSING_FILES}
import org.apache.spark.network.util.ByteUnit
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis.{HintErrorLogger, Resolver}
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
@@ -2899,16 +2900,41 @@ class SQLConf extends Serializable with Logging {
settings.containsKey(key)
}
+ /**
+ * Logs a warning message if the given config key is deprecated.
+ */
+ private def logDeprecationWarning(key: String): Unit = {
+ SQLConf.deprecatedSQLConfigs.get(key).foreach {
+ case DeprecatedConfig(configName, version, comment) =>
+ logWarning(
+ s"The SQL config '$configName' has been deprecated in Spark v$version " +
+ s"and may be removed in the future. $comment")
+ }
+ }
+
+ private def requireDefaultValueOfRemovedConf(key: String, value: String): Unit = {
+ SQLConf.removedSQLConfigs.get(key).foreach {
+ case RemovedConfig(configName, version, defaultValue, comment) =>
+ if (value != defaultValue) {
+ throw new AnalysisException(
+ s"The SQL config '$configName' was removed in the version $version. $comment")
+ }
+ }
+ }
+
protected def setConfWithCheck(key: String, value: String): Unit = {
+ logDeprecationWarning(key)
+ requireDefaultValueOfRemovedConf(key, value)
settings.put(key, value)
}
def unsetConf(key: String): Unit = {
+ logDeprecationWarning(key)
settings.remove(key)
}
def unsetConf(entry: ConfigEntry[_]): Unit = {
- settings.remove(entry.key)
+ unsetConf(entry.key)
}
def clear(): Unit = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
index e1b44b5..e9bc25d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql.internal.SQLConf.{DeprecatedConfig, RemovedConfig}
* @since 2.0.0
*/
@Stable
-class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging {
+class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {
/**
* Sets the given Spark runtime configuration property.
@@ -40,8 +40,6 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging
*/
def set(key: String, value: String): Unit = {
requireNonStaticConf(key)
- requireDefaultValueOfRemovedConf(key, value)
- logDeprecationWarning(key)
sqlConf.setConfString(key, value)
}
@@ -130,7 +128,6 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging
*/
def unset(key: String): Unit = {
requireNonStaticConf(key)
- logDeprecationWarning(key)
sqlConf.unsetConf(key)
}
@@ -161,26 +158,4 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging
throw new AnalysisException(s"Cannot modify the value of a Spark config: $key")
}
}
-
- private def requireDefaultValueOfRemovedConf(key: String, value: String): Unit = {
- SQLConf.removedSQLConfigs.get(key).foreach {
- case RemovedConfig(configName, version, defaultValue, comment) =>
- if (value != defaultValue) {
- throw new AnalysisException(
- s"The SQL config '$configName' was removed in the version $version. $comment")
- }
- }
- }
-
- /**
- * Logs a warning message if the given config key is deprecated.
- */
- private def logDeprecationWarning(key: String): Unit = {
- SQLConf.deprecatedSQLConfigs.get(key).foreach {
- case DeprecatedConfig(configName, version, comment) =>
- logWarning(
- s"The SQL config '$configName' has been deprecated in Spark v$version " +
- s"and may be removed in the future. $comment")
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org