You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2018/12/07 15:56:06 UTC
spark git commit: [SPARK-26060][SQL][FOLLOW-UP] Rename the config
name.
Repository: spark
Updated Branches:
refs/heads/master 477226520 -> 1ab3d3e47
[SPARK-26060][SQL][FOLLOW-UP] Rename the config name.
## What changes were proposed in this pull request?
This is a follow-up of #23031 to rename the config name to `spark.sql.legacy.setCommandRejectsSparkCoreConfs`.
## How was this patch tested?
Existing tests.
Closes #23245 from ueshin/issues/SPARK-26060/rename_config.
Authored-by: Takuya UESHIN <ue...@databricks.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1ab3d3e4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1ab3d3e4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1ab3d3e4
Branch: refs/heads/master
Commit: 1ab3d3e474ce2e36d58aea8ad09fb61f0c73e5c5
Parents: 4772265
Author: Takuya UESHIN <ue...@databricks.com>
Authored: Fri Dec 7 07:55:54 2018 -0800
Committer: Dongjoon Hyun <do...@apache.org>
Committed: Fri Dec 7 07:55:54 2018 -0800
----------------------------------------------------------------------
docs/sql-migration-guide-upgrade.md | 2 +-
.../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 7 ++++---
.../src/main/scala/org/apache/spark/sql/RuntimeConfig.scala | 2 +-
3 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1ab3d3e4/docs/sql-migration-guide-upgrade.md
----------------------------------------------------------------------
diff --git a/docs/sql-migration-guide-upgrade.md b/docs/sql-migration-guide-upgrade.md
index 3638b08..67c30fb 100644
--- a/docs/sql-migration-guide-upgrade.md
+++ b/docs/sql-migration-guide-upgrade.md
@@ -31,7 +31,7 @@ displayTitle: Spark SQL Upgrading Guide
- In Spark version 2.4 and earlier, partition column value is converted as null if it can't be casted to corresponding user provided schema. Since 3.0, partition column value is validated with user provided schema. An exception is thrown if the validation fails. You can disable such validation by setting `spark.sql.sources.validatePartitionColumns` to `false`.
- - In Spark version 2.4 and earlier, the `SET` command works without any warnings even if the specified key is for `SparkConf` entries and it has no effect because the command does not update `SparkConf`, but the behavior might confuse users. Since 3.0, the command fails if a `SparkConf` key is used. You can disable such a check by setting `spark.sql.legacy.execution.setCommandRejectsSparkConfs` to `false`.
+ - In Spark version 2.4 and earlier, the `SET` command works without any warnings even if the specified key is for `SparkConf` entries and it has no effect because the command does not update `SparkConf`, but the behavior might confuse users. Since 3.0, the command fails if a `SparkConf` key is used. You can disable such a check by setting `spark.sql.legacy.setCommandRejectsSparkCoreConfs` to `false`.
- Spark applications which are built with Spark version 2.4 and prior, and call methods of `UserDefinedFunction`, need to be re-compiled with Spark 3.0, as they are not binary compatible with Spark 3.0.
http://git-wip-us.apache.org/repos/asf/spark/blob/1ab3d3e4/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 6857b8d..86e068b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1621,8 +1621,8 @@ object SQLConf {
.intConf
.createWithDefault(25)
- val SET_COMMAND_REJECTS_SPARK_CONFS =
- buildConf("spark.sql.legacy.execution.setCommandRejectsSparkConfs")
+ val SET_COMMAND_REJECTS_SPARK_CORE_CONFS =
+ buildConf("spark.sql.legacy.setCommandRejectsSparkCoreConfs")
.internal()
.doc("If it is set to true, SET command will fail when the key is registered as " +
"a SparkConf entry.")
@@ -2057,7 +2057,8 @@ class SQLConf extends Serializable with Logging {
def maxToStringFields: Int = getConf(SQLConf.MAX_TO_STRING_FIELDS)
- def setCommandRejectsSparkConfs: Boolean = getConf(SQLConf.SET_COMMAND_REJECTS_SPARK_CONFS)
+ def setCommandRejectsSparkCoreConfs: Boolean =
+ getConf(SQLConf.SET_COMMAND_REJECTS_SPARK_CORE_CONFS)
def legacyTimeParserEnabled: Boolean = getConf(SQLConf.LEGACY_TIME_PARSER_ENABLED)
http://git-wip-us.apache.org/repos/asf/spark/blob/1ab3d3e4/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
index d83a01f..0f5aab7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
@@ -153,7 +153,7 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {
if (SQLConf.staticConfKeys.contains(key)) {
throw new AnalysisException(s"Cannot modify the value of a static config: $key")
}
- if (sqlConf.setCommandRejectsSparkConfs &&
+ if (sqlConf.setCommandRejectsSparkCoreConfs &&
ConfigEntry.findEntry(key) != null && !SQLConf.sqlConfEntries.containsKey(key)) {
throw new AnalysisException(s"Cannot modify the value of a Spark config: $key")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org