You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2017/05/20 04:10:07 UTC
[3/3] hive git commit: BUG-48761 : Invalid caller context from Hive
provided to HDFS (Sergey Shelukhin, reviewed by Hitesh Shah)
BUG-48761 : Invalid caller context from Hive provided to HDFS (Sergey Shelukhin, reviewed by Hitesh Shah)
Conflicts:
ql/src/java/org/apache/hadoop/hive/ql/Driver.java
service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4ce7a0ad
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4ce7a0ad
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4ce7a0ad
Branch: refs/heads/branch-2.2
Commit: 4ce7a0ad43431f400dcebd45eb68ff9da2b3bf73
Parents: f4b676c
Author: Jason Dere <jd...@hortonworks.com>
Authored: Fri Dec 16 14:39:51 2016 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri May 19 16:35:24 2017 -0700
----------------------------------------------------------------------
.../hive/ql/exec/tez/TezSessionState.java | 13 +++++++++++
.../service/cli/session/HiveSessionImpl.java | 2 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 23 +++++++++++++++++---
.../apache/hadoop/hive/shims/HadoopShims.java | 10 +++++++++
4 files changed, 44 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4ce7a0ad/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index cc22af3..7889128 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -219,6 +220,16 @@ public class TezSessionState {
protected void openInternal(final HiveConf conf, Collection<String> additionalFiles,
boolean isAsync, LogHelper console, Path scratchDir) throws IOException, LoginException,
IllegalArgumentException, URISyntaxException, TezException {
+ LOG.info("Opening the session with id " + sessionId + " for thread "
+ + Thread.currentThread().getName() + " log trace id - " + conf.getLogIdVar(SessionState.get().getSessionId())
+ + " query id - " + conf.getVar(HiveConf.ConfVars.HIVEQUERYID));
+ String queryId = conf.getVar(HiveConf.ConfVars.HIVEQUERYID);
+ if ((queryId == null) || (queryId.isEmpty())) {
+ ShimLoader.getHadoopShims().setHadoopSessionContext(sessionId);
+ } else {
+ ShimLoader.getHadoopShims().setHadoopQueryContext(queryId);
+ }
+
this.conf = conf;
// TODO Why is the queue name set again. It has already been setup via setQueueName. Do only one of the two.
String confQueueName = conf.get(TezConfiguration.TEZ_QUEUE_NAME);
@@ -434,6 +445,8 @@ public class TezSessionState {
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
+ // reset caller context
+ ShimLoader.getHadoopShims().setHadoopCallerContext("");
}
private void setupSessionAcls(Configuration tezConf, HiveConf hiveConf) throws
http://git-wip-us.apache.org/repos/asf/hive/blob/4ce7a0ad/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 4d5cdac..759c14f 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -368,7 +368,7 @@ public class HiveSessionImpl implements HiveSession {
// set the log context for debugging
LOG.info("We are setting the hadoop caller context to " + sessionState.getSessionId()
+ " for thread " + Thread.currentThread().getName());
- ShimLoader.getHadoopShims().setHadoopCallerContext(sessionState.getSessionId());
+ ShimLoader.getHadoopShims().setHadoopSessionContext(sessionState.getSessionId());
Hive.set(sessionHive);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4ce7a0ad/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 7d31f03..290b136 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1057,7 +1057,6 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
}
-
public static class StoragePolicyShim implements HadoopShims.StoragePolicyShim {
private final DistributedFileSystem dfs;
@@ -1384,7 +1383,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
try {
Subject origSubject = (Subject) getSubjectMethod.invoke(baseUgi);
-
+
Subject subject = new Subject(false, origSubject.getPrincipals(),
cloneCredentials(origSubject.getPublicCredentials()),
cloneCredentials(origSubject.getPrivateCredentials()));
@@ -1402,12 +1401,30 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
return set;
}
-
+
public void setHadoopCallerContext(String callerContext) {
CallerContext.setCurrent(new CallerContext.Builder(callerContext).build());
}
@Override
+ public void setHadoopQueryContext(final String callerContext) {
+ String context = new String(callerContext);
+ if (!context.startsWith("HIVE_QUERY_ID")) {
+ context = "HIVE_QUERY_ID:" + context;
+ }
+ setHadoopCallerContext(context);
+ }
+
+ @Override
+ public void setHadoopSessionContext(final String sessionId) {
+ String context = new String(sessionId);
+ if (!context.startsWith("HIVE_SSN_ID")) {
+ context = "HIVE_SSN_ID:" + context;
+ }
+ setHadoopCallerContext(context);
+ }
+
+ @Override
public String getHadoopCallerContext() {
if (CallerContext.getCurrent() == null) {
return "";
http://git-wip-us.apache.org/repos/asf/hive/blob/4ce7a0ad/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 3887474..1f10e13 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -672,4 +672,14 @@ public interface HadoopShims {
* get current caller context of HDFS and Yarn.
*/
public String getHadoopCallerContext();
+
+ /*
+ * Set up context specific caller context with query prefix.
+ */
+ void setHadoopQueryContext(String queryId);
+
+ /*
+ * Set up context specific caller context with session prefix.
+ */
+ void setHadoopSessionContext(String sessionId);
}