You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by to...@apache.org on 2017/03/07 14:41:25 UTC
crunch git commit: CRUNCH-638: Improve dot file generation for better supportability. Contributed by Gergő Pásztor.
Repository: crunch
Updated Branches:
refs/heads/master e3ba6fcd0 -> e520d9f6e
CRUNCH-638: Improve dot file generation for better supportability. Contributed by Gerg\u0151 P�sztor.
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/e520d9f6
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/e520d9f6
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/e520d9f6
Branch: refs/heads/master
Commit: e520d9f6e55bdcd16e5ad9715f46ddce4ce96f76
Parents: e3ba6fc
Author: Tom White <to...@apache.org>
Authored: Tue Mar 7 14:38:52 2017 +0000
Committer: Tom White <to...@apache.org>
Committed: Tue Mar 7 14:38:52 2017 +0000
----------------------------------------------------------------------
.../apache/crunch/impl/mr/plan/DotfilesIT.java | 3 +-
.../apache/crunch/impl/mr/plan/DotfileUtil.java | 30 ++++++++++++++++----
.../apache/crunch/impl/mr/plan/MSCRPlanner.java | 5 ++--
3 files changed, 29 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/e520d9f6/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java b/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java
index c33348a..99e25b7 100644
--- a/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java
@@ -116,10 +116,11 @@ public class DotfilesIT {
String[] dotfileNames = dotfileNames(dotfileDir.getRootFile());
- assertEquals(5, dotfileNames.length);
+ assertEquals(6, dotfileNames.length);
assertTrue(containsFileEndingWith(dotfileNames, "jobplan.dot"));
assertTrue(containsFileEndingWith(dotfileNames, "split_graph_plan.dot"));
+ assertTrue(containsFileEndingWith(dotfileNames, "split_graph_with_components_plan.dot"));
assertTrue(containsFileEndingWith(dotfileNames, "rt_plan.dot"));
assertTrue(containsFileEndingWith(dotfileNames, "base_graph_plan.dot"));
assertTrue(containsFileEndingWith(dotfileNames, "lineage_plan.dot"));
http://git-wip-us.apache.org/repos/asf/crunch/blob/e520d9f6/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java
index 6475e87..ee854d9 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java
@@ -47,6 +47,7 @@ public class DotfileUtil {
private String rtNodesDotfile = "";
private String basePlanGraphDotfile = "";
private String splitGraphPlanDotfile = "";
+ private String splitGraphWithComponentsPlanDotfile = "";
private String pcollectionLineageDotfile = "";
private String planDotFile = "";
@@ -75,8 +76,22 @@ public class DotfileUtil {
void buildBaseGraphDotfile(Map<PCollectionImpl<?>, Set<Target>> outputs, Graph graph) {
if (isDebugDotfilesEnabled(conf)) {
try {
- basePlanGraphDotfile = new DotfileWriterGraph(graph, outputs, null).buildDiagram("Base Graph ("
- + jarClass.getSimpleName() + ")");
+ basePlanGraphDotfile = new DotfileWriterGraph(graph, outputs, null)
+ .buildDiagram("Base Graph (" + jarClass.getSimpleName() + ")");
+ } catch (Exception ex) {
+ LOG.error("Problem creating debug dotfile:", ex);
+ }
+ }
+ }
+
+ /**
+ * Builds the split graph dotfile only if the dotfile-debug mode is enabled.
+ */
+ void buildSplitGraphDotfile(Map<PCollectionImpl<?>, Set<Target>> outputs, Graph graph) {
+ if (isDebugDotfilesEnabled(conf)) {
+ try {
+ splitGraphPlanDotfile = new DotfileWriterGraph(graph, outputs, null)
+ .buildDiagram("Split Graph (" + jarClass.getSimpleName() + ")");
} catch (Exception ex) {
LOG.error("Problem creating debug dotfile:", ex);
}
@@ -84,13 +99,15 @@ public class DotfileUtil {
}
/**
- * Builds a split graph dotfile only if the dotfile-debug mode is enabled.
+ * Builds a split graph with components dotfile only if the dotfile-debug mode is enabled.
*/
- void buildSplitGraphDotfile(Map<PCollectionImpl<?>, Set<Target>> outputs, Graph graph, List<List<Vertex>> components) {
+ void buildSplitGraphWithComponentsDotfile(
+ Map<PCollectionImpl<?>, Set<Target>> outputs, Graph graph, List<List<Vertex>> components
+ ) {
if (isDebugDotfilesEnabled(conf)) {
try {
- splitGraphPlanDotfile = new DotfileWriterGraph(graph, outputs, components)
- .buildDiagram("Graph With Components (" + jarClass.getSimpleName() + ")");
+ splitGraphWithComponentsPlanDotfile = new DotfileWriterGraph(graph, outputs, components)
+ .buildDiagram("Split Graph With Components (" + jarClass.getSimpleName() + ")");
} catch (Exception ex) {
LOG.error("Problem creating debug dotfile:", ex);
}
@@ -145,6 +162,7 @@ public class DotfileUtil {
exec.addNamedDotFile("rt_plan", rtNodesDotfile);
exec.addNamedDotFile("base_graph_plan", basePlanGraphDotfile);
exec.addNamedDotFile("split_graph_plan", splitGraphPlanDotfile);
+ exec.addNamedDotFile("split_graph_with_components_plan", splitGraphWithComponentsPlanDotfile);
exec.addNamedDotFile("lineage_plan", pcollectionLineageDotfile);
}
} catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/crunch/blob/e520d9f6/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java
index eace851..5113a2b 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java
@@ -137,8 +137,9 @@ public class MSCRPlanner {
List<List<Vertex>> components = graph.connectedComponents();
// Generate the debug graph dotfiles (if configuration is enabled)
- dotfileUtil.buildBaseGraphDotfile(outputs, graph);
- dotfileUtil.buildSplitGraphDotfile(outputs, graph, components);
+ dotfileUtil.buildBaseGraphDotfile(outputs, baseGraph);
+ dotfileUtil.buildSplitGraphDotfile(outputs, graph);
+ dotfileUtil.buildSplitGraphWithComponentsDotfile(outputs, graph, components);
// For each component, we will create one or more job prototypes,
// depending on its profile.