You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/05/15 20:27:32 UTC
spark git commit: [CORE] Protect additional test vars from early GC
Repository: spark
Updated Branches:
refs/heads/master b1b9d5802 -> 270d4b518
[CORE] Protect additional test vars from early GC
Fix more places in which some test variables could be collected early by aggressive JVM optimization.
Added a couple of comments to note where existing references are sufficient in the same test pattern.
Author: Tim Ellison <t....@gmail.com>
Closes #6187 from tellison/DefeatEarlyGC and squashes the following commits:
27329d9 [Tim Ellison] [CORE] Protect additional test vars from early GC
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/270d4b51
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/270d4b51
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/270d4b51
Branch: refs/heads/master
Commit: 270d4b5181b95e3f1f131b1d65dde00a7e5b9d6e
Parents: b1b9d58
Author: Tim Ellison <t....@gmail.com>
Authored: Fri May 15 11:27:24 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Fri May 15 11:27:24 2015 -0700
----------------------------------------------------------------------
.../test/scala/org/apache/spark/ContextCleanerSuite.scala | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/270d4b51/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
index cb30e1f..0922a2c 100644
--- a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
@@ -165,6 +165,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
}
// Test that GC causes RDD cleanup after dereferencing the RDD
+ // Note rdd is used after previous GC to avoid early collection by the JVM
val postGCTester = new CleanerTester(sc, rddIds = Seq(rdd.id))
rdd = null // Make RDD out of scope
runGC()
@@ -181,9 +182,9 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
intercept[Exception] {
preGCTester.assertCleanup()(timeout(1000 millis))
}
+ rdd.count() // Defeat early collection by the JVM
// Test that GC causes shuffle cleanup after dereferencing the RDD
- rdd.count() // Defeat any early collection of rdd variable by the JVM
val postGCTester = new CleanerTester(sc, shuffleIds = Seq(0))
rdd = null // Make RDD out of scope, so that corresponding shuffle goes out of scope
runGC()
@@ -201,6 +202,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
}
// Test that GC causes broadcast cleanup after dereferencing the broadcast variable
+ // Note broadcast is used after previous GC to avoid early collection by the JVM
val postGCTester = new CleanerTester(sc, broadcastIds = Seq(broadcast.id))
broadcast = null // Make broadcast variable out of scope
runGC()
@@ -226,7 +228,7 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
// the checkpoint is not cleaned by default (without the configuration set)
var postGCTester = new CleanerTester(sc, Seq(rddId), Nil, Nil, Nil)
- rdd = null // Make RDD out of scope
+ rdd = null // Make RDD out of scope, ok if collected earlier
runGC()
postGCTester.assertCleanup()
assert(fs.exists(RDDCheckpointData.rddCheckpointDataPath(sc, rddId).get))
@@ -245,6 +247,9 @@ class ContextCleanerSuite extends ContextCleanerSuiteBase {
// Confirm the checkpoint directory exists
assert(fs.exists(RDDCheckpointData.rddCheckpointDataPath(sc, rddId).get))
+ // Reference rdd to defeat any early collection by the JVM
+ rdd.count()
+
// Test that GC causes checkpoint data cleanup after dereferencing the RDD
postGCTester = new CleanerTester(sc, Seq(rddId), Nil, Nil, Seq(rddId))
rdd = null // Make RDD out of scope
@@ -352,6 +357,7 @@ class SortShuffleContextCleanerSuite extends ContextCleanerSuiteBase(classOf[Sor
intercept[Exception] {
preGCTester.assertCleanup()(timeout(1000 millis))
}
+ rdd.count() // Defeat early collection by the JVM
// Test that GC causes shuffle cleanup after dereferencing the RDD
val postGCTester = new CleanerTester(sc, shuffleIds = Seq(0))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org