You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/05/03 04:38:01 UTC
spark git commit: [SPARK-14685][CORE] Document heritability of
localProperties
Repository: spark
Updated Branches:
refs/heads/master 4e3685ae5 -> 8028f3a0b
[SPARK-14685][CORE] Document heritability of localProperties
## What changes were proposed in this pull request?
This updates the java-/scala- doc for setLocalProperty to document heritability of localProperties. This also adds tests for that behaviour.
## How was this patch tested?
Tests pass. New tests were added.
Author: Marcin Tustin <ma...@gmail.com>
Closes #12455 from marcintustin/SPARK-14685.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8028f3a0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8028f3a0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8028f3a0
Branch: refs/heads/master
Commit: 8028f3a0b4003af15ed44d9ef4727b56f4b10534
Parents: 4e3685a
Author: Marcin Tustin <ma...@gmail.com>
Authored: Mon May 2 19:37:57 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Mon May 2 19:37:57 2016 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/SparkContext.scala | 5 ++++
.../spark/api/java/JavaSparkContext.scala | 9 +++++--
.../org/apache/spark/SparkContextSuite.scala | 28 ++++++++++++++++++++
3 files changed, 40 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8028f3a0/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 302dec2..58618b4 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -608,6 +608,11 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* scheduler pool. User-defined properties may also be set here. These properties are propagated
* through to worker tasks and can be accessed there via
* [[org.apache.spark.TaskContext#getLocalProperty]].
+ *
+ * These properties are inherited by child threads spawned from this thread. This
+ * may have unexpected consequences when working with thread pools. The standard java
+ * implementation of thread pools have worker threads spawn other worker threads.
+ * As a result, local properties may propagate unpredictably.
*/
def setLocalProperty(key: String, value: String) {
if (value == null) {
http://git-wip-us.apache.org/repos/asf/spark/blob/8028f3a0/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index dfd91ae..fb63234 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@ -712,8 +712,13 @@ class JavaSparkContext(val sc: SparkContext)
}
/**
- * Set a local property that affects jobs submitted from this thread, such as the
- * Spark fair scheduler pool.
+ * Set a local property that affects jobs submitted from this thread, and all child
+ * threads, such as the Spark fair scheduler pool.
+ *
+ * These properties are inherited by child threads spawned from this thread. This
+ * may have unexpected consequences when working with thread pools. The standard java
+ * implementation of thread pools have worker threads spawn other worker threads.
+ * As a result, local properties may propagate unpredictably.
*/
def setLocalProperty(key: String, value: String): Unit = sc.setLocalProperty(key, value)
http://git-wip-us.apache.org/repos/asf/spark/blob/8028f3a0/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index a759f36..6398708 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -323,4 +323,32 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext {
assert(sc.getConf.getInt("spark.executor.instances", 0) === 6)
}
}
+
+
+ test("localProperties are inherited by spawned threads.") {
+ sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
+ sc.setLocalProperty("testProperty", "testValue")
+ var result = "unset";
+ val thread = new Thread() { override def run() = {result = sc.getLocalProperty("testProperty")}}
+ thread.start()
+ thread.join()
+ sc.stop()
+ assert(result == "testValue")
+ }
+
+ test("localProperties do not cross-talk between threads.") {
+ sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
+ var result = "unset";
+ val thread1 = new Thread() {
+ override def run() = {sc.setLocalProperty("testProperty", "testValue")}}
+ // testProperty should be unset and thus return null
+ val thread2 = new Thread() {
+ override def run() = {result = sc.getLocalProperty("testProperty")}}
+ thread1.start()
+ thread1.join()
+ thread2.start()
+ thread2.join()
+ sc.stop()
+ assert(result == null)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org