You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2014/03/25 00:58:53 UTC
git commit: TEZ-876. Fix graphviz dag generation for Hive. (hitesh)
Repository: incubator-tez
Updated Branches:
refs/heads/master a3a9ceaad -> 908512fc3
TEZ-876. Fix graphviz dag generation for Hive. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/908512fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/908512fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/908512fc
Branch: refs/heads/master
Commit: 908512fc378d5dc6982ac2516822709f0cf06b0f
Parents: a3a9cea
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon Mar 24 16:58:37 2014 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon Mar 24 16:58:37 2014 -0700
----------------------------------------------------------------------
.../apache/tez/dag/api/TezConfiguration.java | 2 +-
.../org/apache/tez/dag/app/DAGAppMaster.java | 39 ++++++++++++++------
2 files changed, 28 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/908512fc/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index a5ba9e3..3d42577 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -347,7 +347,7 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_GENERATE_DAG_VIZ =
TEZ_PREFIX + "generate.dag.viz";
- public static final boolean TEZ_GENERATE_DAG_VIZ_DEFAULT = false;
+ public static final boolean TEZ_GENERATE_DAG_VIZ_DEFAULT = true;
/**
* Comma separated list of containers which should be profiled.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/908512fc/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index cb9f2fc..45a4f18 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -41,6 +41,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
@@ -169,6 +171,8 @@ public class DAGAppMaster extends AbstractService {
*/
public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private static Pattern sanitizeLabelPattern = Pattern.compile("[:\\-\\W]+");
+
private Clock clock;
private final boolean isSession;
private long appsStartTime;
@@ -635,26 +639,34 @@ public class DAGAppMaster extends AbstractService {
return className;
}
+
+ private String sanitizeLabelForViz(String label) {
+ Matcher m = sanitizeLabelPattern.matcher(label);
+ return m.replaceAll("_");
+ }
+
private void generateDAGVizFile(TezDAGID dagId, DAGPlan dagPB) {
- Graph graph = new Graph(dagPB.getName());
+ Graph graph = new Graph(sanitizeLabelForViz(dagPB.getName()));
for (VertexPlan v : dagPB.getVertexList()) {
- String nodeLabel = v.getName().replace("\\W","_")
+ String nodeLabel = sanitizeLabelForViz(v.getName())
+ "[" + getShortClassName(v.getProcessorDescriptor().getClassName() + "]");
- Graph.Node n = graph.newNode(v.getName(), nodeLabel);
+ Graph.Node n = graph.newNode(sanitizeLabelForViz(v.getName()), nodeLabel);
for (DAGProtos.RootInputLeafOutputProto input : v.getInputsList()) {
- Graph.Node inputNode = graph.getNode(v.getName()
- + "_" + input.getName());
- inputNode.setLabel(v.getName() + "[" + input.getName() + "]");
+ Graph.Node inputNode = graph.getNode(sanitizeLabelForViz(v.getName())
+ + "_" + sanitizeLabelForViz(input.getName()));
+ inputNode.setLabel(sanitizeLabelForViz(v.getName())
+ + "[" + sanitizeLabelForViz(input.getName()) + "]");
inputNode.setShape("box");
inputNode.addEdge(n, "Input"
+ " [inputClass=" + getShortClassName(input.getEntityDescriptor().getClassName())
+ ", initializer=" + getShortClassName(input.getInitializerClassName()) + "]");
}
for (DAGProtos.RootInputLeafOutputProto output : v.getOutputsList()) {
- Graph.Node outputNode = graph.getNode(v.getName()
- + "_" + output.getName());
- outputNode.setLabel(v.getName() + "[" + output.getName() + "]");
+ Graph.Node outputNode = graph.getNode(sanitizeLabelForViz(v.getName())
+ + "_" + sanitizeLabelForViz(output.getName()));
+ outputNode.setLabel(sanitizeLabelForViz(v.getName())
+ + "[" + sanitizeLabelForViz(output.getName()) + "]");
outputNode.setShape("box");
n.addEdge(outputNode, "Output"
+ " [outputClass=" + getShortClassName(output.getEntityDescriptor().getClassName())
@@ -664,8 +676,8 @@ public class DAGAppMaster extends AbstractService {
for (DAGProtos.EdgePlan e : dagPB.getEdgeList()) {
- Graph.Node n = graph.getNode(e.getInputVertexName());
- n.addEdge(graph.getNode(e.getOutputVertexName()),
+ Graph.Node n = graph.getNode(sanitizeLabelForViz(e.getInputVertexName()));
+ n.addEdge(graph.getNode(sanitizeLabelForViz(e.getOutputVertexName())),
"["
+ "input=" + getShortClassName(e.getEdgeSource().getClassName())
+ ", output=" + getShortClassName(e.getEdgeDestination().getClassName())
@@ -687,8 +699,11 @@ public class DAGAppMaster extends AbstractService {
outputFile += dagId.toString() + ".dot";
try {
+ LOG.info("Generating DAG graphviz file"
+ + ", dagId=" + dagId.toString()
+ + ", filePath=" + outputFile);
graph.save(outputFile);
- } catch (IOException e) {
+ } catch (Exception e) {
LOG.warn("Error occurred when trying to save graph structure"
+ " for dag " + dagId.toString(), e);
}