You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2020/06/03 08:19:11 UTC
[spark] branch branch-3.0 updated: [SPARK-31886][WEBUI] Fix the
wrong coloring of nodes in DAG-viz
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new f6a77c3 [SPARK-31886][WEBUI] Fix the wrong coloring of nodes in DAG-viz
f6a77c3 is described below
commit f6a77c31fc47ea4e3c40576ed587b4a75ecaeda1
Author: Kousuke Saruta <sa...@oss.nttdata.com>
AuthorDate: Wed Jun 3 01:15:36 2020 -0700
[SPARK-31886][WEBUI] Fix the wrong coloring of nodes in DAG-viz
### What changes were proposed in this pull request?
This PR fixes a wrong coloring issue in the DAG-viz.
In the Job Page and Stage Page, nodes which are associated with "barrier mode" in the DAG-viz will be colored pale green.
But, with some type of jobs, nodes which are not associated with the mode will also colored.
You can reproduce with the following operation.
```
sc.parallelize(1 to 10).barrier.mapPartitions(identity).repartition(1).collect()
```
<img width="376" alt="wrong-coloring" src="https://user-images.githubusercontent.com/4736016/83403670-1711df00-a444-11ea-9457-c683f75bc566.png">
In the screen shot above, `repartition` in `Stage 1` is not associated with barrier mode so the corresponding node should not be colored pale green.
The cause of this issue is the logic which chooses HTML elements to be colored is wrong.
The logic chooses such elements based on whether each element is associated with a style class (`clusterId` in the code).
But when an operation crosses over shuffle (like `repartition` above), a `clusterId` can be duplicated and non-barrier mode node is also associated with the same `clusterId`.
### Why are the changes needed?
This is a bug.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Newly added test case with the following command.
```
build/sbt -Dtest.default.exclude.tags= -Dspark.test.webdriver.chrome.driver=/path/to/chromedriver "testOnly org.apache.spark.ui.ChromeUISeleniumSuite -- -z SPARK-31886"
```
Closes #28694 from sarutak/fix-wrong-barrier-color.
Authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
Signed-off-by: Gengliang Wang <ge...@databricks.com>
(cherry picked from commit 8ed93c9355bc2af6fe456d88aa693c8db69d0bbf)
Signed-off-by: Gengliang Wang <ge...@databricks.com>
---
.../org/apache/spark/ui/static/spark-dag-viz.js | 8 ++++---
.../spark/ui/RealBrowserUISeleniumSuite.scala | 28 ++++++++++++++++++++++
2 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
index fd4a48d..474c453 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
@@ -173,9 +173,11 @@ function renderDagViz(forJob) {
});
metadataContainer().selectAll(".barrier-rdd").each(function() {
- var rddId = d3.select(this).text().trim();
- var clusterId = VizConstants.clusterPrefix + rddId;
- svg.selectAll("g." + clusterId).classed("barrier", true)
+ var opId = d3.select(this).text().trim();
+ var opClusterId = VizConstants.clusterPrefix + opId;
+ var stageId = $(this).parents(".stage-metadata").attr("stage-id");
+ var stageClusterId = VizConstants.graphPrefix + stageId;
+ svg.selectAll("g[id=" + stageClusterId + "] g." + opClusterId).classed("barrier", true)
});
resizeSvg(svg);
diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
index 06b6483..4b018f6 100644
--- a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
@@ -100,6 +100,34 @@ abstract class RealBrowserUISeleniumSuite(val driverProp: String)
}
}
+ test("SPARK-31886: Color barrier execution mode RDD correctly") {
+ withSpark(newSparkContext()) { sc =>
+ sc.parallelize(1 to 10).barrier.mapPartitions(identity).repartition(1).collect()
+
+ eventually(timeout(10.seconds), interval(50.milliseconds)) {
+ goToUi(sc, "/jobs/job/?id=0")
+ webDriver.findElement(By.id("job-dag-viz")).click()
+
+ val stage0 = webDriver.findElement(By.cssSelector("g[id='graph_0']"))
+ val stage1 = webDriver.findElement(By.cssSelector("g[id='graph_1']"))
+ val barrieredOps = webDriver.findElements(By.className("barrier-rdd")).iterator()
+
+ while (barrieredOps.hasNext) {
+ val barrieredOpId = barrieredOps.next().getAttribute("innerHTML")
+ val foundInStage0 =
+ stage0.findElements(
+ By.cssSelector("g.barrier.cluster.cluster_" + barrieredOpId))
+ assert(foundInStage0.size === 1)
+
+ val foundInStage1 =
+ stage1.findElements(
+ By.cssSelector("g.barrier.cluster.cluster_" + barrieredOpId))
+ assert(foundInStage1.size === 0)
+ }
+ }
+ }
+ }
+
/**
* Create a test SparkContext with the SparkUI enabled.
* It is safe to `get` the SparkUI directly from the SparkContext returned here.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org