You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2015/02/06 20:15:04 UTC
spark git commit: [SPARK-5555] Enable UISeleniumSuite tests
Repository: spark
Updated Branches:
refs/heads/master 32e964c41 -> 0d74bd7fd
[SPARK-5555] Enable UISeleniumSuite tests
This patch enables UISeleniumSuite, a set of tests for the Spark application web UI. These tests were previously disabled because they were slow, but I think we now have sufficient test time budget that the benefit of enabling them outweighs the time costs.
Author: Josh Rosen <jo...@databricks.com>
Closes #4334 from JoshRosen/enable-uiseleniumsuite and squashes the following commits:
4ab9477 [Josh Rosen] Use BeforeAndAfterAll to cleanup WebDriver
71efc72 [Josh Rosen] Update broken UISeleniumSuite tests; use random port #.
a5ab595 [Josh Rosen] Enable UISeleniumSuite tests.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0d74bd7f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0d74bd7f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0d74bd7f
Branch: refs/heads/master
Commit: 0d74bd7fd7b2722d08eddc5c269b8b2b6cb47635
Parents: 32e964c
Author: Josh Rosen <jo...@databricks.com>
Authored: Fri Feb 6 11:14:58 2015 -0800
Committer: Josh Rosen <jo...@databricks.com>
Committed: Fri Feb 6 11:14:58 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/ui/UISeleniumSuite.scala | 29 ++++++++++++++------
1 file changed, 20 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0d74bd7f/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index e85a436..6a97238 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -32,12 +32,21 @@ import org.apache.spark.api.java.StorageLevels
import org.apache.spark.shuffle.FetchFailedException
/**
- * Selenium tests for the Spark Web UI. These tests are not run by default
- * because they're slow.
+ * Selenium tests for the Spark Web UI.
*/
-@DoNotDiscover
-class UISeleniumSuite extends FunSuite with WebBrowser with Matchers {
- implicit val webDriver: WebDriver = new HtmlUnitDriver
+class UISeleniumSuite extends FunSuite with WebBrowser with Matchers with BeforeAndAfterAll {
+
+ implicit var webDriver: WebDriver = _
+
+ override def beforeAll(): Unit = {
+ webDriver = new HtmlUnitDriver
+ }
+
+ override def afterAll(): Unit = {
+ if (webDriver != null) {
+ webDriver.quit()
+ }
+ }
/**
* Create a test SparkContext with the SparkUI enabled.
@@ -48,6 +57,7 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers {
.setMaster("local")
.setAppName("test")
.set("spark.ui.enabled", "true")
+ .set("spark.ui.port", "0")
val sc = new SparkContext(conf)
assert(sc.ui.isDefined)
sc
@@ -93,7 +103,7 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers {
}
eventually(timeout(5 seconds), interval(50 milliseconds)) {
go to (sc.ui.get.appUIAddress.stripSuffix("/") + "/stages")
- find(id("active")).get.text should be("Active Stages (0)")
+ find(id("active")) should be(None) // Since we hide empty tables
find(id("failed")).get.text should be("Failed Stages (1)")
}
@@ -105,7 +115,7 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers {
}
eventually(timeout(5 seconds), interval(50 milliseconds)) {
go to (sc.ui.get.appUIAddress.stripSuffix("/") + "/stages")
- find(id("active")).get.text should be("Active Stages (0)")
+ find(id("active")) should be(None) // Since we hide empty tables
// The failure occurs before the stage becomes active, hence we should still show only one
// failed stage, not two:
find(id("failed")).get.text should be("Failed Stages (1)")
@@ -167,13 +177,14 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers {
test("job progress bars should handle stage / task failures") {
withSpark(newSparkContext()) { sc =>
- val data = sc.parallelize(Seq(1, 2, 3)).map(identity).groupBy(identity)
+ val data = sc.parallelize(Seq(1, 2, 3), 1).map(identity).groupBy(identity)
val shuffleHandle =
data.dependencies.head.asInstanceOf[ShuffleDependency[_, _, _]].shuffleHandle
// Simulate fetch failures:
val mappedData = data.map { x =>
val taskContext = TaskContext.get
- if (taskContext.attemptNumber == 0) { // Cause this stage to fail on its first attempt.
+ if (taskContext.taskAttemptId() == 1) {
+ // Cause the post-shuffle stage to fail on its first attempt with a single task failure
val env = SparkEnv.get
val bmAddress = env.blockManager.blockManagerId
val shuffleId = shuffleHandle.shuffleId
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org