You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/03/28 00:31:07 UTC
svn commit: r1669710 - in
/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez:
TezJobMonitor.java TezTask.java
Author: prasanthj
Date: Fri Mar 27 23:31:07 2015
New Revision: 1669710
URL: http://svn.apache.org/r1669710
Log:
HIVE-10125: LLAP: Print execution modes in tez in-place UI (Prasanth Jayachandran)
Modified:
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java?rev=1669710&r1=1669709&r2=1669710&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java Fri Mar 27 23:31:07 2015
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.exec.Ma
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.tez.common.counters.TaskCounter;
@@ -73,17 +74,17 @@ import jline.TerminalFactory;
public class TezJobMonitor {
private static final String CLASS_NAME = TezJobMonitor.class.getName();
- private static final int MIN_TERMINAL_WIDTH = 80;
+ private static final int MIN_TERMINAL_WIDTH = 89;
private static final int COLUMN_1_WIDTH = 16;
- private static final int SEPARATOR_WIDTH = 80;
+ private static final int SEPARATOR_WIDTH = MIN_TERMINAL_WIDTH;
// keep this within 80 chars width. If more columns needs to be added then update min terminal
// width requirement and separator width accordingly
- private static final String HEADER_FORMAT = "%16s%12s %5s %9s %7s %7s %6s %6s";
- private static final String VERTEX_FORMAT = "%-16s%12s %5s %9s %7s %7s %6s %6s";
+ private static final String HEADER_FORMAT = "%16s%11s %9s %5s %9s %7s %7s %6s %6s";
+ private static final String VERTEX_FORMAT = "%-16s%11s %9s %5s %9s %7s %7s %6s %6s";
private static final String FOOTER_FORMAT = "%-15s %-30s %-4s %-25s";
private static final String HEADER = String.format(HEADER_FORMAT,
- "VERTICES", "STATUS", "TOTAL", "COMPLETED", "RUNNING", "PENDING", "FAILED", "KILLED");
+ "VERTICES", "EXECUTOR", "STATUS", "TOTAL", "COMPLETED", "RUNNING", "PENDING", "FAILED", "KILLED");
// method and dag summary format
private static final String SUMMARY_HEADER_FORMAT = "%-16s %-12s %-12s %-12s %-19s %-19s %-15s %-15s %-15s";
@@ -114,6 +115,7 @@ public class TezJobMonitor {
private final NumberFormat secondsFormat;
private final NumberFormat commaFormat;
private static final List<DAGClient> shutdownList;
+ private Map<String, BaseWork> workMap;
static {
shutdownList = Collections.synchronizedList(new LinkedList<DAGClient>());
@@ -140,7 +142,8 @@ public class TezJobMonitor {
"Shutdown hook was not properly initialized");
}
- public TezJobMonitor() {
+ public TezJobMonitor(Map<String, BaseWork> workMap) {
+ this.workMap = workMap;
console = SessionState.getConsole();
secondsFormat = new DecimalFormat("#0.00");
commaFormat = NumberFormat.getNumberInstance(Locale.US);
@@ -672,10 +675,12 @@ public class TezJobMonitor {
}
}
- // Map 1 .......... SUCCEEDED 7 7 0 0 0 0
+ // Map 1 .......... container SUCCEEDED 7 7 0 0 0 0
String nameWithProgress = getNameWithProgress(s, complete, total);
+ String mode = getMode(s, workMap);
String vertexStr = String.format(VERTEX_FORMAT,
nameWithProgress,
+ mode,
vertexState.toString(),
total,
complete,
@@ -701,6 +706,21 @@ public class TezJobMonitor {
reprintLine(separator);
}
+ private String getMode(String name, Map<String, BaseWork> workMap) {
+ String mode = "container";
+ BaseWork work = workMap.get(name);
+ if (work != null) {
+ if (work.getLlapMode()) {
+ mode = "llap";
+ } else if (work.getUberMode()) {
+ mode = "uber";
+ } else {
+ mode = "container";
+ }
+ }
+ return mode;
+ }
+
// Map 1 ..........
private String getNameWithProgress(String s, int complete, int total) {
String result = "";
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1669710&r1=1669709&r2=1669710&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Fri Mar 27 23:31:07 2015
@@ -27,9 +27,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -48,7 +48,6 @@ import org.apache.hadoop.hive.ql.plan.Un
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -59,7 +58,6 @@ import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.GroupInputEdge;
import org.apache.tez.dag.api.SessionNotRunning;
-import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.VertexGroup;
import org.apache.tez.dag.api.client.DAGClient;
@@ -163,7 +161,7 @@ public class TezTask extends Task<TezWor
additionalLr, inputOutputJars, inputOutputLocalResources);
// finally monitor will print progress until the job is done
- TezJobMonitor monitor = new TezJobMonitor();
+ TezJobMonitor monitor = new TezJobMonitor(work.getWorkMap());
rc = monitor.monitorExecution(client, ctx.getHiveTxnManager(), conf, dag);
// fetch the counters