You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2015/01/22 01:52:37 UTC
spark git commit: [SPARK-5355] make SparkConf thread-safe
Repository: spark
Updated Branches:
refs/heads/master 3be2a887b -> 9bad06226
[SPARK-5355] make SparkConf thread-safe
The SparkConf is not thread-safe, but is accessed by many threads. The getAll() could return parts of the configs if another thread is access it.
This PR changes SparkConf.settings to a thread-safe TrieMap.
Author: Davies Liu <da...@databricks.com>
Closes #4143 from davies/safe-conf and squashes the following commits:
f8fa1cf [Davies Liu] change to TrieMap
a1d769a [Davies Liu] make SparkConf thread-safe
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9bad0622
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9bad0622
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9bad0622
Branch: refs/heads/master
Commit: 9bad062268676aaa66dcbddd1e0ab7f2d7742425
Parents: 3be2a88
Author: Davies Liu <da...@databricks.com>
Authored: Wed Jan 21 16:51:42 2015 -0800
Committer: Josh Rosen <jo...@databricks.com>
Committed: Wed Jan 21 16:51:54 2015 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/SparkConf.scala | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9bad0622/core/src/main/scala/org/apache/spark/SparkConf.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index a0ce107..f9d4aa4 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -18,6 +18,7 @@
package org.apache.spark
import scala.collection.JavaConverters._
+import scala.collection.concurrent.TrieMap
import scala.collection.mutable.{HashMap, LinkedHashSet}
import org.apache.spark.serializer.KryoSerializer
@@ -46,7 +47,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging {
/** Create a SparkConf that loads defaults from system properties and the classpath */
def this() = this(true)
- private[spark] val settings = new HashMap[String, String]()
+ private[spark] val settings = new TrieMap[String, String]()
if (loadDefaults) {
// Load any spark.* system properties
@@ -177,7 +178,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging {
}
/** Get all parameters as a list of pairs */
- def getAll: Array[(String, String)] = settings.clone().toArray
+ def getAll: Array[(String, String)] = settings.toArray
/** Get a parameter as an integer, falling back to a default if not set */
def getInt(key: String, defaultValue: Int): Int = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org