You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2014/11/17 20:22:27 UTC
[18/50] [abbrv] tez git commit: TEZ-1735. Allow setting basic info
per DAG for Tez UI. (hitesh)
TEZ-1735. Allow setting basic info per DAG for Tez UI. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/db4112cc
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/db4112cc
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/db4112cc
Branch: refs/heads/TEZ-8
Commit: db4112cc0019b36c587503ad5673b4a7265cc91a
Parents: 5782716
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon Nov 3 19:55:57 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon Nov 3 19:55:57 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../main/java/org/apache/tez/dag/api/DAG.java | 26 +++++++++++++++++---
tez-api/src/main/proto/DAGApiRecords.proto | 1 +
.../apache/tez/dag/history/utils/DAGUtils.java | 4 +++
.../tez/dag/history/utils/TestDAGUtils.java | 4 +++
.../examples/TestOrderedWordCount.java | 1 +
6 files changed, 34 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f741109..6fab873 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -98,6 +98,7 @@ ALL CHANGES:
TEZ-1726. Build broken against Hadoop-2.6.0 due to change in NodeReport.
TEZ-1579. MR examples should be setting mapreduce.framework.name to yarn-tez.
TEZ-1731. OnDiskMerger can end up clobbering files across tasks with LocalDiskFetch enabled.
+ TEZ-1735. Allow setting basic info per DAG for Tez UI.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 34c27bf..acef4da 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -48,7 +48,6 @@ import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
import org.apache.tez.dag.api.VertexGroup.GroupInfo;
-import org.apache.tez.dag.api.TaskLocationHint;
import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
@@ -87,7 +86,8 @@ public class DAG {
Set<GroupInputEdge> groupInputEdges = Sets.newHashSet();
private DAGAccessControls dagAccessControls;
Map<String, LocalResource> commonTaskLocalFiles = Maps.newHashMap();
-
+ String dagInfo;
+
private Stack<String> topologicalVertexStack = new Stack<String>();
private DAG(String name) {
@@ -148,7 +148,24 @@ public class DAG {
this.credentials = credentials;
return this;
}
-
+
+ /**
+ * Set description info for this DAG that can be used for visualization purposes.
+ * @param dagInfo JSON blob as a serialized string.
+ * Recognized keys by the UI are:
+ * "context" - The application context in which this DAG is being used.
+ * For example, this could be set to "Hive" or "Pig" if
+ * this is being run as part of a Hive or Pig script.
+ * "description" - General description on what this DAG is going to do.
+ * In the case of Hive, this could be the SQL query text.
+ * @return {@link DAG}
+ */
+ public synchronized DAG setDAGInfo(String dagInfo) {
+ Preconditions.checkNotNull(dagInfo);
+ this.dagInfo = dagInfo;
+ return this;
+ }
+
/**
* Create a group of vertices that share a common output. This can be used to implement
* unions efficiently.
@@ -589,6 +606,9 @@ public class DAG {
DAGPlan.Builder dagBuilder = DAGPlan.newBuilder();
dagBuilder.setName(this.name);
+ if (this.dagInfo != null && !this.dagInfo.isEmpty()) {
+ dagBuilder.setDagInfo(this.dagInfo);
+ }
if (!vertexGroups.isEmpty()) {
for (VertexGroup av : vertexGroups) {
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-api/src/main/proto/DAGApiRecords.proto
----------------------------------------------------------------------
diff --git a/tez-api/src/main/proto/DAGApiRecords.proto b/tez-api/src/main/proto/DAGApiRecords.proto
index 177faba..0539405 100644
--- a/tez-api/src/main/proto/DAGApiRecords.proto
+++ b/tez-api/src/main/proto/DAGApiRecords.proto
@@ -164,6 +164,7 @@ message DAGPlan {
optional bytes credentials_binary = 5;
repeated PlanVertexGroupInfo vertex_groups = 6;
repeated PlanLocalResource local_resource = 7;
+ optional string dag_info = 8;
}
// DAG monitoring messages
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index 0bcbcbe..5f259989 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -49,6 +49,7 @@ import org.codehaus.jettison.json.JSONObject;
public class DAGUtils {
public static final String DAG_NAME_KEY = "dagName";
+ public static final String DAG_INFO_KEY = "dagInfo";
public static final String VERTICES_KEY = "vertices";
public static final String EDGES_KEY = "edges";
public static final String VERTEX_GROUPS_KEY = "vertexGroups";
@@ -138,6 +139,9 @@ public class DAGUtils {
final int version = 1;
Map<String,Object> dagMap = new LinkedHashMap<String, Object>();
dagMap.put(DAG_NAME_KEY, dagPlan.getName());
+ if (dagPlan.hasDagInfo()) {
+ dagMap.put(DAG_INFO_KEY, dagPlan.getDagInfo());
+ }
dagMap.put(VERSION_KEY, version);
ArrayList<Object> verticesList = new ArrayList<Object>();
for (DAGProtos.VertexPlan vertexPlan : dagPlan.getVertexList()) {
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
index 6f4202a..496a12c 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
@@ -71,6 +71,7 @@ public class TestDAGUtils {
dummyTaskCount, dummyTaskResource);
DAG dag = DAG.create("testDag");
+ dag.setDAGInfo("dagInfo");
String groupName1 = "uv12";
org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
OutputDescriptor outDesc = OutputDescriptor.create("output.class")
@@ -110,6 +111,9 @@ public class TestDAGUtils {
Map<String, Object> atsMap = DAGUtils.convertDAGPlanToATSMap(dagPlan);
Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_NAME_KEY));
+ Assert.assertEquals("testDag", atsMap.get(DAGUtils.DAG_NAME_KEY));
+ Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_INFO_KEY));
+ Assert.assertEquals("dagInfo", atsMap.get(DAGUtils.DAG_INFO_KEY));
Assert.assertEquals(dagPlan.getName(), atsMap.get(DAGUtils.DAG_NAME_KEY));
Assert.assertTrue(atsMap.containsKey("version"));
Assert.assertEquals(1, atsMap.get("version"));
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
index e519085..ec6b429 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
@@ -270,6 +270,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
vertices.add(finalReduceVertex);
DAG dag = DAG.create("OrderedWordCount" + dagIndex);
+ dag.setDAGInfo("{ \"context\": \"Tez\", \"description\": \"TestOrderedWordCount Job\" }");
for (int i = 0; i < vertices.size(); ++i) {
dag.addVertex(vertices.get(i));
}