You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/11/27 04:07:28 UTC
[1/2] git commit: [SPARK-963] Wait for SparkListenerBus eventQueue to
be empty before checking jobLogger state
Updated Branches:
refs/heads/master cb976dfb5 -> 615213fb8
[SPARK-963] Wait for SparkListenerBus eventQueue to be empty before checking jobLogger state
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/ed7ecb93
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/ed7ecb93
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/ed7ecb93
Branch: refs/heads/master
Commit: ed7ecb93ce6ce259eae1f5aeb28e9e336fafa30f
Parents: cb976df
Author: Mark Hamstra <ma...@gmail.com>
Authored: Tue Nov 26 13:30:17 2013 -0800
Committer: Mark Hamstra <ma...@gmail.com>
Committed: Tue Nov 26 13:30:17 2013 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/scheduler/JobLoggerSuite.scala | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ed7ecb93/core/src/test/scala/org/apache/spark/scheduler/JobLoggerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/JobLoggerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/JobLoggerSuite.scala
index 9848818..002368f 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/JobLoggerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/JobLoggerSuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.rdd.RDD
class JobLoggerSuite extends FunSuite with LocalSparkContext with ShouldMatchers {
+ val WAIT_TIMEOUT_MILLIS = 10000
test("inner method") {
sc = new SparkContext("local", "joblogger")
@@ -92,6 +93,8 @@ class JobLoggerSuite extends FunSuite with LocalSparkContext with ShouldMatchers
val rdd = sc.parallelize(1 to 1e2.toInt, 4).map{ i => (i % 12, 2 * i) }
rdd.reduceByKey(_+_).collect()
+ assert(sc.dagScheduler.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS))
+
val user = System.getProperty("user.name", SparkContext.SPARK_UNKNOWN_USER)
joblogger.getLogDir should be ("/tmp/spark-%s".format(user))
@@ -120,7 +123,9 @@ class JobLoggerSuite extends FunSuite with LocalSparkContext with ShouldMatchers
sc.addSparkListener(joblogger)
val rdd = sc.parallelize(1 to 1e2.toInt, 4).map{ i => (i % 12, 2 * i) }
rdd.reduceByKey(_+_).collect()
-
+
+ assert(sc.dagScheduler.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS))
+
joblogger.onJobStartCount should be (1)
joblogger.onJobEndCount should be (1)
joblogger.onTaskEndCount should be (8)
[2/2] git commit: Merge pull request #212 from markhamstra/SPARK-963
Posted by ma...@apache.org.
Merge pull request #212 from markhamstra/SPARK-963
[SPARK-963] Fixed races in JobLoggerSuite
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/615213fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/615213fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/615213fb
Branch: refs/heads/master
Commit: 615213fb82812003529367b2a8dff25b66362a1a
Parents: cb976df ed7ecb9
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Tue Nov 26 19:07:20 2013 -0800
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Tue Nov 26 19:07:20 2013 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/scheduler/JobLoggerSuite.scala | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------