You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2017/05/20 04:10:05 UTC
[1/3] hive git commit: HIVE-16402 : Upgrade to Hadoop 2.8.0 (Sahil
Takiar via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-2.2 f7280d02f -> 4ce7a0ad4
HIVE-16402 : Upgrade to Hadoop 2.8.0 (Sahil Takiar via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7ea8e1cb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7ea8e1cb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7ea8e1cb
Branch: refs/heads/branch-2.2
Commit: 7ea8e1cbb4b37bdc62530fbf1f2ad7fe16ea1967
Parents: f7280d0
Author: Sahil Takiar <ta...@gmail.com>
Authored: Sun Apr 9 18:25:27 2017 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri May 19 15:51:19 2017 -0700
----------------------------------------------------------------------
hcatalog/core/pom.xml | 10 ++-
llap-server/pom.xml | 8 ++-
metastore/pom.xml | 16 +++++
pom.xml | 65 ++++++++++++++++++--
.../encrypted/encryption_move_tbl.q.out | 2 +-
shims/0.23/pom.xml | 4 ++
shims/scheduler/pom.xml | 6 ++
storage-api/pom.xml | 2 +-
8 files changed, 103 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7ea8e1cb/hcatalog/core/pom.xml
----------------------------------------------------------------------
diff --git a/hcatalog/core/pom.xml b/hcatalog/core/pom.xml
index 506bf22..92475b4 100644
--- a/hcatalog/core/pom.xml
+++ b/hcatalog/core/pom.xml
@@ -191,7 +191,7 @@
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
- </dependency>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
@@ -208,13 +208,19 @@
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
- </dependency>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
<version>${hadoop.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.pig</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/7ea8e1cb/llap-server/pom.xml
----------------------------------------------------------------------
diff --git a/llap-server/pom.xml b/llap-server/pom.xml
index 22d17b7..da75a7b 100644
--- a/llap-server/pom.xml
+++ b/llap-server/pom.xml
@@ -165,6 +165,10 @@
<version>${slider.version}</version>
<exclusions>
<exclusion>
+ <groupId>asm</groupId>
+ <artifactId>asm</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
@@ -209,8 +213,8 @@
<artifactId>jettison</artifactId>
</exclusion>
<exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/hive/blob/7ea8e1cb/metastore/pom.xml
----------------------------------------------------------------------
diff --git a/metastore/pom.xml b/metastore/pom.xml
index eabcdb9..b6e3add 100644
--- a/metastore/pom.xml
+++ b/metastore/pom.xml
@@ -148,6 +148,22 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${hadoop.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<optional>true</optional>
http://git-wip-us.apache.org/repos/asf/hive/blob/7ea8e1cb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index aed3373..6f4c153 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,7 +137,8 @@
<druid.version>0.9.2</druid.version>
<guava.version>14.0.1</guava.version>
<groovy.version>2.4.4</groovy.version>
- <hadoop.version>2.7.2</hadoop.version>
+ <h2database.version>1.3.166</h2database.version>
+ <hadoop.version>2.8.0</hadoop.version>
<hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path>
<hbase.version>1.1.1</hbase.version>
<!-- required for logging test to avoid including hbase which pulls disruptor transitively -->
@@ -651,13 +652,24 @@
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
- </dependency>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
- <exclusion>
+ <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
@@ -697,7 +709,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
@@ -712,10 +724,55 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>${hbase.version}</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/7ea8e1cb/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
index 55eb22e..5cea8f2 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
@@ -49,7 +49,7 @@ PREHOOK: query: ALTER TABLE default.encrypted_table RENAME TO encrypted_db.encry
PREHOOK: type: ALTERTABLE_RENAME
PREHOOK: Input: default@encrypted_table
PREHOOK: Output: default@encrypted_table
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Alter Table operation for default.encrypted_table failed to move data due to: '/build/ql/test/data/warehouse/default/encrypted_table can't be moved from encryption zone /build/ql/test/data/warehouse/default/encrypted_table to encryption zone /build/ql/test/data/warehouse/encrypted_db.' See hive log file for details.
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Alter Table operation for default.encrypted_table failed to move data due to: '/build/ql/test/data/warehouse/encrypted_table can't be moved into an encryption zone.' See hive log file for details.
PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
PREHOOK: Input: database:default
http://git-wip-us.apache.org/repos/asf/hive/blob/7ea8e1cb/shims/0.23/pom.xml
----------------------------------------------------------------------
diff --git a/shims/0.23/pom.xml b/shims/0.23/pom.xml
index d0d1d5f..3ff75ba 100644
--- a/shims/0.23/pom.xml
+++ b/shims/0.23/pom.xml
@@ -179,6 +179,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hive/blob/7ea8e1cb/shims/scheduler/pom.xml
----------------------------------------------------------------------
diff --git a/shims/scheduler/pom.xml b/shims/scheduler/pom.xml
index 9141c1e..6cb53bb 100644
--- a/shims/scheduler/pom.xml
+++ b/shims/scheduler/pom.xml
@@ -76,6 +76,12 @@
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<version>${hadoop.version}</version>
<optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/7ea8e1cb/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/storage-api/pom.xml b/storage-api/pom.xml
index 097efdb..b96714e 100644
--- a/storage-api/pom.xml
+++ b/storage-api/pom.xml
@@ -32,7 +32,7 @@
<properties>
<commons-lang.version>2.6</commons-lang.version>
<guava.version>14.0.1</guava.version>
- <hadoop.version>2.7.2</hadoop.version>
+ <hadoop.version>2.8.0</hadoop.version>
<junit.version>4.11</junit.version>
<slf4j.version>1.7.10</slf4j.version>
</properties>
[2/3] hive git commit: HIVE-12387: Issues in Hive's use of
CallerContext (Vikram Dixit K, reviewed by Gunther Hagleitner)
Posted by om...@apache.org.
HIVE-12387: Issues in Hive's use of CallerContext (Vikram Dixit K, reviewed by Gunther Hagleitner)
Conflicts:
cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
ql/src/java/org/apache/hadoop/hive/ql/Driver.java
ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f4b676cd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f4b676cd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f4b676cd
Branch: refs/heads/branch-2.2
Commit: f4b676cd62b304b1926615ec3b53bd65dac7c3ad
Parents: 7ea8e1c
Author: Jason Dere <jd...@hortonworks.com>
Authored: Fri Dec 16 10:57:35 2016 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri May 19 16:22:52 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/Driver.java | 39 ++++++++++++++++----
.../org/apache/hadoop/hive/ql/QueryPlan.java | 35 ++++++++++++++++--
.../apache/hadoop/hive/ql/hooks/ATSHook.java | 25 ++++++++-----
.../parse/TestUpdateDeleteSemanticAnalyzer.java | 2 +-
.../service/cli/session/HiveSessionImpl.java | 10 ++++-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 16 +++++++-
.../apache/hadoop/hive/shims/HadoopShims.java | 10 +++++
7 files changed, 114 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f4b676cd/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 08bd040..c8e2c4b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -112,6 +112,8 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobClient;
@@ -396,12 +398,13 @@ public class Driver implements CommandProcessor {
}
if (ctx != null && ctx.getExplainAnalyze() != AnalyzeState.RUNNING) {
- close();
+ closeInProcess(false);
}
+
if (isInterrupted()) {
return handleInterruption("at beginning of compilation."); //indicate if need clean resource
}
-
+
if (resetTaskIds) {
TaskFactory.resetId();
}
@@ -416,6 +419,8 @@ public class Driver implements CommandProcessor {
SessionState.get().setupQueryCurrentTimestamp();
+ String originalCallerContext = "";
+ HadoopShims shim = ShimLoader.getHadoopShims();
boolean compileError = false;
try {
// Initialize the transaction manager. This must be done before analyze is called.
@@ -437,6 +442,13 @@ public class Driver implements CommandProcessor {
};
ShutdownHookManager.addShutdownHook(shutdownRunner, SHUTDOWN_HOOK_PRIORITY);
+ // we set the hadoop caller context to the query id as soon as we have one.
+ // initially, the caller context is the session id (when creating temp directories)
+ originalCallerContext = shim.getHadoopCallerContext();
+ LOG.info("We are setting the hadoop caller context from " + originalCallerContext + " to "
+ + queryId);
+ shim.setHadoopQueryContext(queryId);
+
if (isInterrupted()) {
return handleInterruption("before parsing and analysing the query");
}
@@ -444,7 +456,7 @@ public class Driver implements CommandProcessor {
if (ctx == null) {
ctx = new Context(conf);
}
-
+
ctx.setTryCount(getTryCount());
ctx.setCmd(command);
ctx.setHDFSCleanup(true);
@@ -502,7 +514,9 @@ public class Driver implements CommandProcessor {
// get the output schema
schema = getSchema(sem, conf);
plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId,
- queryState.getHiveOperation(), schema);
+ queryState.getHiveOperation(), schema,
+ SessionState.get().getSessionId(), Thread.currentThread().getName(),
+ HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_LOG_TRACE_ID));
conf.setQueryString(queryStr);
@@ -598,6 +612,10 @@ public class Driver implements CommandProcessor {
} else {
LOG.info("Completed compiling command(queryId=" + queryId + "); Time taken: " + duration + " seconds");
}
+
+ // reset the caller id.
+ LOG.info("We are resetting the hadoop caller context to " + originalCallerContext);
+ shim.setHadoopCallerContext(originalCallerContext);
}
}
@@ -687,7 +705,7 @@ public class Driver implements CommandProcessor {
}
// The following union operation returns a union, which traverses over the
- // first set once and then then over each element of second set, in order,
+ // first set once and then then over each element of second set, in order,
// that is not contained in first. This means it doesn't replace anything
// in first set, and would preserve the WriteType in WriteEntity in first
// set in case of outputs list.
@@ -1009,7 +1027,7 @@ public class Driver implements CommandProcessor {
conf.set(ValidTxnList.VALID_TXNS_KEY, txnStr);
if(plan.getFetchTask() != null) {
/**
- * This is needed for {@link HiveConf.ConfVars.HIVEFETCHTASKCONVERSION} optimization which
+ * This is needed for {@link HiveConf.ConfVars.HIVEFETCHTASKCONVERSION} optimization which
* initializes JobConf in FetchOperator before recordValidTxns() but this has to be done
* after locks are acquired to avoid race conditions in ACID.
*/
@@ -1663,8 +1681,13 @@ public class Driver implements CommandProcessor {
maxthreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.EXECPARALLETHREADNUMBER);
HookContext hookContext = null;
+
+ String originalCallerContext = "";
boolean executionError = false;
try {
+ LOG.info("Setting caller context to query id " + queryId);
+ originalCallerContext = ShimLoader.getHadoopShims().getHadoopCallerContext();
+ ShimLoader.getHadoopShims().setHadoopQueryContext(queryId);
LOG.info("Executing command(queryId=" + queryId + "): " + queryStr);
// compile and execute can get called from different threads in case of HS2
// so clear timing in this thread's Hive object before proceeding.
@@ -1908,6 +1931,8 @@ public class Driver implements CommandProcessor {
+ org.apache.hadoop.util.StringUtils.stringifyException(e));
return (12);
} finally {
+ LOG.info("Resetting the caller context to " + originalCallerContext);
+ ShimLoader.getHadoopShims().setHadoopCallerContext(originalCallerContext);
if (SessionState.get() != null) {
SessionState.get().getHiveHistory().endQuery(queryId);
}
@@ -2348,7 +2373,7 @@ public class Driver implements CommandProcessor {
this.operationId = opId;
}
- /**
+ /**
* Resets QueryState to get new queryId on Driver reuse.
*/
public void resetQueryState() {
http://git-wip-us.apache.org/repos/asf/hive/blob/f4b676cd/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
index e8c8ae6..d05e53b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
@@ -106,6 +106,9 @@ public class QueryPlan implements Serializable {
private transient Long queryStartTime;
private final HiveOperation operation;
private Boolean autoCommitValue;
+ private String sessionId;
+ private String threadName;
+ private String userProvidedContext;
public QueryPlan() {
this.reducerTimeStatsPerJobList = new ArrayList<ReducerTimeStatsPerJob>();
@@ -113,7 +116,8 @@ public class QueryPlan implements Serializable {
}
public QueryPlan(String queryString, BaseSemanticAnalyzer sem, Long startTime, String queryId,
- HiveOperation operation, Schema resultSchema) {
+ HiveOperation operation, Schema resultSchema,
+ String sessionId, String threadName, String userProvidedContext) {
this.queryString = queryString;
rootTasks = new ArrayList<Task<? extends Serializable>>(sem.getAllRootTasks());
@@ -136,6 +140,9 @@ public class QueryPlan implements Serializable {
this.operation = operation;
this.autoCommitValue = sem.getAutoCommitValue();
this.resultSchema = resultSchema;
+ this.setSessionId(sessionId);
+ this.setThreadName(threadName);
+ this.setUserProvidedContext(userProvidedContext);
}
public String getQueryStr() {
@@ -609,7 +616,6 @@ public class QueryPlan implements Serializable {
try {
q.write(oprot);
} catch (TException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
return q.toString();
}
@@ -623,7 +629,6 @@ public class QueryPlan implements Serializable {
try {
q.write(oprot);
} catch (TException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
return q.toString();
}
@@ -802,4 +807,28 @@ public class QueryPlan implements Serializable {
public Boolean getAutoCommitValue() {
return autoCommitValue;
}
+
+ public String getSessionId() {
+ return sessionId;
+ }
+
+ public void setSessionId(String sessionId) {
+ this.sessionId = sessionId;
+ }
+
+ public String getThreadName() {
+ return threadName;
+ }
+
+ public void setThreadName(String threadName) {
+ this.threadName = threadName;
+ }
+
+ public String getUserProvidedContext() {
+ return userProvidedContext;
+ }
+
+ public void setUserProvidedContext(String userProvidedContext) {
+ this.userProvidedContext = userProvidedContext;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4b676cd/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
index 218e0b4..f968da6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
@@ -79,8 +79,8 @@ public class ATSHook implements ExecuteWithHookContext {
private static final String DEFAULT_ATS_DOMAIN = "hive_default_ats";
private enum OtherInfoTypes {
- QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, SESSION_ID, THREAD_NAME, VERSION,
- CLIENT_IP_ADDRESS, HIVE_ADDRESS, HIVE_INSTANCE_TYPE, CONF, PERF,
+ QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, SESSION_ID, LOG_TRACE_ID, THREAD_NAME, VERSION,
+ CLIENT_IP_ADDRESS, HIVE_ADDRESS, HIVE_INSTANCE_TYPE, CONF, PERF, LLAP_APP_ID
};
private enum ExecutionMode {
MR, TEZ, LLAP, SPARK, NONE
@@ -275,8 +275,7 @@ public class ATSHook implements ExecuteWithHookContext {
String query = plan.getQueryStr();
JSONObject explainPlan = explain.getJSONPlan(null,
work);
- String logID = conf.getLogIdVar(hookContext.getSessionId());
- List<String> tablesRead = getTablesFromEntitySet(hookContext
+ List<String> tablesRead = getTablesFromEntitySet(hookContext
.getInputs());
List<String> tablesWritten = getTablesFromEntitySet(hookContext
.getOutputs());
@@ -297,9 +296,9 @@ public class ATSHook implements ExecuteWithHookContext {
hookContext.getIpAddress(),
hiveInstanceAddress, hiveInstanceType,
hookContext.getSessionId(),
- logID,
+ plan.getUserProvidedContext(),
hookContext.getThreadId(), executionMode,
- tablesRead, tablesWritten, conf));
+ tablesRead, tablesWritten, conf, llapId, domainId));
break;
case POST_EXEC_HOOK:
fireAndForget(createPostHookEvent(queryId,
@@ -361,8 +360,10 @@ public class ATSHook implements ExecuteWithHookContext {
TimelineEntity createPreHookEvent(String queryId, String query, JSONObject explainPlan,
long startTime, String user, String requestuser, int numMrJobs, int numTezJobs, String opId,
String clientIpAddress, String hiveInstanceAddress, String hiveInstanceType,
- String sessionID, String logID, String threadId, String executionMode,
- List<String> tablesRead, List<String> tablesWritten, HiveConf conf) throws Exception {
+ String sessionID, String logTraceId, String threadId, String executionMode,
+ List<String> tablesRead, List<String> tablesWritten, HiveConf conf, ApplicationId llapAppId,
+ String domainId)
+ throws Exception {
JSONObject queryObj = new JSONObject(new LinkedHashMap<>());
queryObj.put("queryText", query);
@@ -409,8 +410,10 @@ public class ATSHook implements ExecuteWithHookContext {
atsEntity.addOtherInfo(OtherInfoTypes.TEZ.name(), numTezJobs > 0);
atsEntity.addOtherInfo(OtherInfoTypes.MAPRED.name(), numMrJobs > 0);
atsEntity.addOtherInfo(OtherInfoTypes.SESSION_ID.name(), sessionID);
- atsEntity.addOtherInfo(OtherInfoTypes.INVOKER_INFO.name(), logID);
atsEntity.addOtherInfo(OtherInfoTypes.THREAD_NAME.name(), threadId);
+ if ((logTraceId != null) && (logTraceId.equals("") == false)) {
+ atsEntity.addOtherInfo(OtherInfoTypes.LOG_TRACE_ID.name(), logTraceId);
+ }
atsEntity.addOtherInfo(OtherInfoTypes.VERSION.name(), VERSION);
if (clientIpAddress != null) {
atsEntity.addOtherInfo(OtherInfoTypes.CLIENT_IP_ADDRESS.name(), clientIpAddress);
@@ -418,6 +421,10 @@ public class ATSHook implements ExecuteWithHookContext {
atsEntity.addOtherInfo(OtherInfoTypes.HIVE_ADDRESS.name(), hiveInstanceAddress);
atsEntity.addOtherInfo(OtherInfoTypes.HIVE_INSTANCE_TYPE.name(), hiveInstanceType);
atsEntity.addOtherInfo(OtherInfoTypes.CONF.name(), confObj.toString());
+ if (llapAppId != null) {
+ atsEntity.addOtherInfo(OtherInfoTypes.LLAP_APP_ID.name(), llapAppId.toString());
+ }
+ atsEntity.setDomainId(domainId);
return atsEntity;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4b676cd/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
index a573808..9fa3b25 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
@@ -283,7 +283,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
// validate the plan
sem.validate();
- QueryPlan plan = new QueryPlan(query, sem, 0L, testName, null, null);
+ QueryPlan plan = new QueryPlan(query, sem, 0L, testName, null, null, "", "", "");
return new ReturnInfo(sem, plan);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4b676cd/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 3f9dfdc..4d5cdac 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -355,7 +355,7 @@ public class HiveSessionImpl implements HiveSession {
}
private synchronized void acquireAfterOpLock(boolean userAccess) {
- // Need to make sure that the this HiveServer2's session's SessionState is
+ // Need to make sure that this HiveServer2's session's session state is
// stored in the thread local for the handler thread.
SessionState.setCurrentSessionState(sessionState);
sessionState.setForwardedAddresses(SessionManager.getForwardedAddresses());
@@ -365,6 +365,10 @@ public class HiveSessionImpl implements HiveSession {
}
// set the thread name with the logging prefix.
sessionState.updateThreadName();
+ // set the log context for debugging
+ LOG.info("We are setting the hadoop caller context to " + sessionState.getSessionId()
+ + " for thread " + Thread.currentThread().getName());
+ ShimLoader.getHadoopShims().setHadoopCallerContext(sessionState.getSessionId());
Hive.set(sessionHive);
}
@@ -390,6 +394,10 @@ public class HiveSessionImpl implements HiveSession {
// can be null in-case of junit tests. skip reset.
// reset thread name at release time.
sessionState.resetThreadName();
+ // reset the HDFS caller context.
+ LOG.info("We are resetting the hadoop caller context for thread "
+ + Thread.currentThread().getName());
+ ShimLoader.getHadoopShims().setHadoopCallerContext("");
}
SessionState.detachSession();
http://git-wip-us.apache.org/repos/asf/hive/blob/f4b676cd/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 916a96b..7d31f03 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@ -1383,7 +1384,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
try {
Subject origSubject = (Subject) getSubjectMethod.invoke(baseUgi);
-
+
Subject subject = new Subject(false, origSubject.getPrincipals(),
cloneCredentials(origSubject.getPublicCredentials()),
cloneCredentials(origSubject.getPrivateCredentials()));
@@ -1401,5 +1402,16 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
return set;
}
-
+
+ public void setHadoopCallerContext(String callerContext) {
+ CallerContext.setCurrent(new CallerContext.Builder(callerContext).build());
+ }
+
+ @Override
+ public String getHadoopCallerContext() {
+ if (CallerContext.getCurrent() == null) {
+ return "";
+ }
+ return CallerContext.getCurrent().getContext();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4b676cd/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index b88e8c9..3887474 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -662,4 +662,14 @@ public interface HadoopShims {
/** Clones the UGI and the Subject. */
UserGroupInformation cloneUgi(UserGroupInformation baseUgi) throws IOException;
+
+ /*
+ * Set up the caller context for HDFS and Yarn.
+ */
+ public void setHadoopCallerContext(String callerContext);
+
+ /*
+ * get current caller context of HDFS and Yarn.
+ */
+ public String getHadoopCallerContext();
}
[3/3] hive git commit: BUG-48761 : Invalid caller context from Hive
provided to HDFS (Sergey Shelukhin, reviewed by Hitesh Shah)
Posted by om...@apache.org.
BUG-48761 : Invalid caller context from Hive provided to HDFS (Sergey Shelukhin, reviewed by Hitesh Shah)
Conflicts:
ql/src/java/org/apache/hadoop/hive/ql/Driver.java
service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4ce7a0ad
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4ce7a0ad
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4ce7a0ad
Branch: refs/heads/branch-2.2
Commit: 4ce7a0ad43431f400dcebd45eb68ff9da2b3bf73
Parents: f4b676c
Author: Jason Dere <jd...@hortonworks.com>
Authored: Fri Dec 16 14:39:51 2016 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri May 19 16:35:24 2017 -0700
----------------------------------------------------------------------
.../hive/ql/exec/tez/TezSessionState.java | 13 +++++++++++
.../service/cli/session/HiveSessionImpl.java | 2 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 23 +++++++++++++++++---
.../apache/hadoop/hive/shims/HadoopShims.java | 10 +++++++++
4 files changed, 44 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4ce7a0ad/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index cc22af3..7889128 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -219,6 +220,16 @@ public class TezSessionState {
protected void openInternal(final HiveConf conf, Collection<String> additionalFiles,
boolean isAsync, LogHelper console, Path scratchDir) throws IOException, LoginException,
IllegalArgumentException, URISyntaxException, TezException {
+ LOG.info("Opening the session with id " + sessionId + " for thread "
+ + Thread.currentThread().getName() + " log trace id - " + conf.getLogIdVar(SessionState.get().getSessionId())
+ + " query id - " + conf.getVar(HiveConf.ConfVars.HIVEQUERYID));
+ String queryId = conf.getVar(HiveConf.ConfVars.HIVEQUERYID);
+ if ((queryId == null) || (queryId.isEmpty())) {
+ ShimLoader.getHadoopShims().setHadoopSessionContext(sessionId);
+ } else {
+ ShimLoader.getHadoopShims().setHadoopQueryContext(queryId);
+ }
+
this.conf = conf;
// TODO Why is the queue name set again. It has already been setup via setQueueName. Do only one of the two.
String confQueueName = conf.get(TezConfiguration.TEZ_QUEUE_NAME);
@@ -434,6 +445,8 @@ public class TezSessionState {
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
+ // reset caller context
+ ShimLoader.getHadoopShims().setHadoopCallerContext("");
}
private void setupSessionAcls(Configuration tezConf, HiveConf hiveConf) throws
http://git-wip-us.apache.org/repos/asf/hive/blob/4ce7a0ad/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 4d5cdac..759c14f 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -368,7 +368,7 @@ public class HiveSessionImpl implements HiveSession {
// set the log context for debugging
LOG.info("We are setting the hadoop caller context to " + sessionState.getSessionId()
+ " for thread " + Thread.currentThread().getName());
- ShimLoader.getHadoopShims().setHadoopCallerContext(sessionState.getSessionId());
+ ShimLoader.getHadoopShims().setHadoopSessionContext(sessionState.getSessionId());
Hive.set(sessionHive);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4ce7a0ad/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 7d31f03..290b136 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1057,7 +1057,6 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
}
-
public static class StoragePolicyShim implements HadoopShims.StoragePolicyShim {
private final DistributedFileSystem dfs;
@@ -1384,7 +1383,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
try {
Subject origSubject = (Subject) getSubjectMethod.invoke(baseUgi);
-
+
Subject subject = new Subject(false, origSubject.getPrincipals(),
cloneCredentials(origSubject.getPublicCredentials()),
cloneCredentials(origSubject.getPrivateCredentials()));
@@ -1402,12 +1401,30 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
return set;
}
-
+
public void setHadoopCallerContext(String callerContext) {
CallerContext.setCurrent(new CallerContext.Builder(callerContext).build());
}
@Override
+ public void setHadoopQueryContext(final String callerContext) {
+ String context = new String(callerContext);
+ if (!context.startsWith("HIVE_QUERY_ID")) {
+ context = "HIVE_QUERY_ID:" + context;
+ }
+ setHadoopCallerContext(context);
+ }
+
+ @Override
+ public void setHadoopSessionContext(final String sessionId) {
+ String context = new String(sessionId);
+ if (!context.startsWith("HIVE_SSN_ID")) {
+ context = "HIVE_SSN_ID:" + context;
+ }
+ setHadoopCallerContext(context);
+ }
+
+ @Override
public String getHadoopCallerContext() {
if (CallerContext.getCurrent() == null) {
return "";
http://git-wip-us.apache.org/repos/asf/hive/blob/4ce7a0ad/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 3887474..1f10e13 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -672,4 +672,14 @@ public interface HadoopShims {
* get current caller context of HDFS and Yarn.
*/
public String getHadoopCallerContext();
+
+ /*
+ * Set up context specific caller context with query prefix.
+ */
+ void setHadoopQueryContext(String queryId);
+
+ /*
+ * Set up context specific caller context with session prefix.
+ */
+ void setHadoopSessionContext(String sessionId);
}