You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/07/12 00:38:46 UTC
spark git commit: [SPARK-24761][SQL] Adding of isModifiable() to
RuntimeConfig
Repository: spark
Updated Branches:
refs/heads/master e008ad175 -> 3ab48f985
[SPARK-24761][SQL] Adding of isModifiable() to RuntimeConfig
## What changes were proposed in this pull request?
In the PR, I propose to extend `RuntimeConfig` by new method `isModifiable()` which returns `true` if a config parameter can be modified at runtime (for current session state). For static SQL and core parameters, the method returns `false`.
## How was this patch tested?
Added new test to `RuntimeConfigSuite` for checking Spark core and SQL parameters.
Author: Maxim Gekk <ma...@databricks.com>
Closes #21730 from MaxGekk/is-modifiable.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3ab48f98
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3ab48f98
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3ab48f98
Branch: refs/heads/master
Commit: 3ab48f985c7f96bc9143caad99bf3df7cc984583
Parents: e008ad1
Author: Maxim Gekk <ma...@databricks.com>
Authored: Wed Jul 11 17:38:43 2018 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Wed Jul 11 17:38:43 2018 -0700
----------------------------------------------------------------------
python/pyspark/sql/conf.py | 8 ++++++++
.../scala/org/apache/spark/sql/internal/SQLConf.scala | 4 ++++
.../scala/org/apache/spark/sql/RuntimeConfig.scala | 11 +++++++++++
.../org/apache/spark/sql/RuntimeConfigSuite.scala | 14 ++++++++++++++
4 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3ab48f98/python/pyspark/sql/conf.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/conf.py b/python/pyspark/sql/conf.py
index db49040..f80bf59 100644
--- a/python/pyspark/sql/conf.py
+++ b/python/pyspark/sql/conf.py
@@ -63,6 +63,14 @@ class RuntimeConfig(object):
raise TypeError("expected %s '%s' to be a string (was '%s')" %
(identifier, obj, type(obj).__name__))
+ @ignore_unicode_prefix
+ @since(2.4)
+ def isModifiable(self, key):
+ """Indicates whether the configuration property with the given key
+ is modifiable in the current session.
+ """
+ return self._jconf.isModifiable(key)
+
def _test():
import os
http://git-wip-us.apache.org/repos/asf/spark/blob/3ab48f98/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index ae56cc9..14dd528 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1907,4 +1907,8 @@ class SQLConf extends Serializable with Logging {
}
cloned
}
+
+ def isModifiable(key: String): Boolean = {
+ sqlConfEntries.containsKey(key) && !staticConfKeys.contains(key)
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/3ab48f98/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
index b352e33..3c39579 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
@@ -133,6 +133,17 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {
}
/**
+ * Indicates whether the configuration property with the given key
+ * is modifiable in the current session.
+ *
+ * @return `true` if the configuration property is modifiable. For static SQL, Spark Core,
+ * invalid (not existing) and other non-modifiable configuration properties,
+ * the returned value is `false`.
+ * @since 2.4.0
+ */
+ def isModifiable(key: String): Boolean = sqlConf.isModifiable(key)
+
+ /**
* Returns whether a particular key is set.
*/
protected[sql] def contains(key: String): Boolean = {
http://git-wip-us.apache.org/repos/asf/spark/blob/3ab48f98/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala
index cfe2e9f..cdcea09 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/RuntimeConfigSuite.scala
@@ -54,4 +54,18 @@ class RuntimeConfigSuite extends SparkFunSuite {
conf.get("k1")
}
}
+
+ test("SPARK-24761: is a config parameter modifiable") {
+ val conf = newConf()
+
+ // SQL configs
+ assert(!conf.isModifiable("spark.sql.sources.schemaStringLengthThreshold"))
+ assert(conf.isModifiable("spark.sql.streaming.checkpointLocation"))
+ // Core configs
+ assert(!conf.isModifiable("spark.task.cpus"))
+ assert(!conf.isModifiable("spark.executor.cores"))
+ // Invalid config parameters
+ assert(!conf.isModifiable(""))
+ assert(!conf.isModifiable("invalid config parameter"))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org