You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ad...@apache.org on 2014/07/04 04:24:31 UTC
git commit: [SPARK-1097] Workaround Hadoop conf
ConcurrentModification issue
Repository: spark
Updated Branches:
refs/heads/master fdc4c112e -> 5fa0a0576
[SPARK-1097] Workaround Hadoop conf ConcurrentModification issue
Workaround Hadoop conf ConcurrentModification issue
Author: Raymond Liu <ra...@intel.com>
Closes #1273 from colorant/hadoopRDD and squashes the following commits:
994e98b [Raymond Liu] Address comments
e2cda3d [Raymond Liu] Workaround Hadoop conf ConcurrentModification issue
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5fa0a057
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5fa0a057
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5fa0a057
Branch: refs/heads/master
Commit: 5fa0a05763ab1d527efe20e3b10539ac5ffc36de
Parents: fdc4c11
Author: Raymond Liu <ra...@intel.com>
Authored: Thu Jul 3 19:24:22 2014 -0700
Committer: Aaron Davidson <aa...@databricks.com>
Committed: Thu Jul 3 19:24:22 2014 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5fa0a057/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
index 98dcbf4..0410285 100644
--- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
@@ -141,8 +141,8 @@ class HadoopRDD[K, V](
// local process. The local cache is accessed through HadoopRDD.putCachedMetadata().
// The caching helps minimize GC, since a JobConf can contain ~10KB of temporary objects.
// synchronize to prevent ConcurrentModificationException (Spark-1097, Hadoop-10456)
- broadcastedConf.synchronized {
- val newJobConf = new JobConf(broadcastedConf.value.value)
+ conf.synchronized {
+ val newJobConf = new JobConf(conf)
initLocalJobConfFuncOpt.map(f => f(newJobConf))
HadoopRDD.putCachedMetadata(jobConfCacheKey, newJobConf)
newJobConf