You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/01/27 22:51:18 UTC
[2/2] hive git commit: HIVE-12357 : Allow user to set tez job name
(Gunther Hagleitner, reviewed by Vikram Dixit K, Sergey Shelukhin)
HIVE-12357 : Allow user to set tez job name (Gunther Hagleitner, reviewed by Vikram Dixit K, Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/36f8b759
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/36f8b759
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/36f8b759
Branch: refs/heads/branch-2.0
Commit: 36f8b7590998d5ad95f8fed229469e447883feb7
Parents: e83dd41
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Jan 27 13:45:38 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Jan 27 13:51:04 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
.../apache/hadoop/hive/ql/exec/tez/DagUtils.java | 16 ++++++++++++++++
.../org/apache/hadoop/hive/ql/exec/tez/TezTask.java | 10 ++++++----
.../org/apache/hadoop/hive/ql/hooks/ATSHook.java | 4 +++-
4 files changed, 28 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/36f8b759/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 14b86e3..7a25ece 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1198,6 +1198,9 @@ public class HiveConf extends Configuration {
HIVETEZLOGLEVEL("hive.tez.log.level", "INFO",
"The log level to use for tasks executing as part of the DAG.\n" +
"Used only if hive.tez.java.opts is used to configure Java options."),
+ HIVEQUERYNAME ("hive.query.name", null,
+ "This named is used by Tez to set the dag name. This name in turn will appear on \n" +
+ "the Tez UI representing the work that was done."),
HIVEOPTIMIZEBUCKETINGSORTING("hive.optimize.bucketingsorting", true,
"Don't create a reducer for enforcing \n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/36f8b759/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index e8864ae..63fb37a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
@@ -1226,6 +1227,21 @@ public class DagUtils {
}
}
+ public String createDagName(Configuration conf, QueryPlan plan) {
+ String name = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYNAME);
+
+ if (name == null) {
+ name = conf.get("mapred.job.name");
+ }
+
+ if (name == null) {
+ name = plan.getQueryId();
+ }
+
+ assert name != null;
+ return name;
+ }
+
private DagUtils() {
// don't instantiate
}
http://git-wip-us.apache.org/repos/asf/hive/blob/36f8b759/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 3cb7439..83defea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -176,9 +176,8 @@ public class TezTask extends Task<TezWork> {
if (driverContext.getCtx() == null) {
boolean a = false;
}
- CallerContext callerContext = CallerContext.create("HIVE",
- conf.getLogIdVar(SessionState.get().getSessionId()) + " "
- + conf.getVar(HiveConf.ConfVars.HIVEQUERYID),
+ CallerContext callerContext = CallerContext.create(
+ "HIVE", queryPlan.getQueryId(),
"HIVE_QUERY_ID", queryPlan.getQueryStr());
dag.setCallerContext(callerContext);
@@ -320,7 +319,10 @@ public class TezTask extends Task<TezWork> {
FileSystem fs = scratchDir.getFileSystem(conf);
// the name of the dag is what is displayed in the AM/Job UI
- DAG dag = DAG.create(work.getName());
+ String dagName = utils.createDagName(conf, queryPlan);
+
+ LOG.info("Dag name: " + dagName);
+ DAG dag = DAG.create(dagName);
// set some info for the query
JSONObject json = new JSONObject(new LinkedHashMap()).put("context", "Hive")
http://git-wip-us.apache.org/repos/asf/hive/blob/36f8b759/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
index 0be8b3c..f490161 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
@@ -50,13 +50,14 @@ public class ATSHook implements ExecuteWithHookContext {
private static final Logger LOG = LoggerFactory.getLogger(ATSHook.class.getName());
private static final Object LOCK = new Object();
+ private static final int VERSION = 2;
private static ExecutorService executor;
private static TimelineClient timelineClient;
private enum EntityTypes { HIVE_QUERY_ID };
private enum EventTypes { QUERY_SUBMITTED, QUERY_COMPLETED };
private enum OtherInfoTypes {
- QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, THREAD_NAME
+ QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, THREAD_NAME, VERSION
};
private enum PrimaryFilterTypes { user, requestuser, operationid };
private static final int WAIT_TIME = 3;
@@ -193,6 +194,7 @@ public class ATSHook implements ExecuteWithHookContext {
atsEntity.addOtherInfo(OtherInfoTypes.MAPRED.name(), numMrJobs > 0);
atsEntity.addOtherInfo(OtherInfoTypes.INVOKER_INFO.name(), logID);
atsEntity.addOtherInfo(OtherInfoTypes.THREAD_NAME.name(), Thread.currentThread().getName());
+ atsEntity.addOtherInfo(OtherInfoTypes.VERSION.name(), VERSION);
return atsEntity;
}