You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2018/10/15 13:13:15 UTC
hive git commit: HIVE-12254: Improve logging with yarn/hdfs (Aihua Xu
via Peter Vary)
Repository: hive
Updated Branches:
refs/heads/master 15c53c4c9 -> 8f3b312e0
HIVE-12254: Improve logging with yarn/hdfs (Aihua Xu via Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8f3b312e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8f3b312e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8f3b312e
Branch: refs/heads/master
Commit: 8f3b312e049c3892b3fd80e009b2c1aa8869e8ea
Parents: 15c53c4
Author: Aihua Xu <ai...@apache.org>
Authored: Mon Oct 15 15:12:34 2018 +0200
Committer: Peter Vary <pv...@cloudera.com>
Committed: Mon Oct 15 15:12:34 2018 +0200
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/cli/CliDriver.java | 6 ++++++
.../hadoop/hive/cli/TestCliDriverMethods.java | 3 +++
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 1 +
ql/src/java/org/apache/hadoop/hive/ql/IDriver.java | 2 ++
.../apache/hadoop/hive/ql/reexec/ReExecDriver.java | 5 +++++
.../hadoop/hive/ql/session/SessionState.java | 2 ++
.../hive/service/cli/operation/Operation.java | 4 ++++
.../hive/service/cli/operation/SQLOperation.java | 3 +++
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 17 ++++++++++++++++-
.../org/apache/hadoop/hive/shims/HadoopShims.java | 16 ++++++++++++++++
10 files changed, 58 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
----------------------------------------------------------------------
diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
index 806663d..2e7adae 100644
--- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
+++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
@@ -78,6 +78,7 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hive.common.util.HiveStringUtils;
import org.apache.hive.common.util.ShutdownHookManager;
@@ -236,9 +237,13 @@ public class CliDriver {
out.println(cmd);
}
+ // Set HDFS CallerContext to queryId and reset back to sessionId after the query is done
+ ShimLoader.getHadoopShims().setHadoopQueryContext(qp.getQueryState().getQueryId());
ret = qp.run(cmd).getResponseCode();
+
if (ret != 0) {
qp.close();
+ ShimLoader.getHadoopShims().setHadoopSessionContext(ss.getSessionId());
return ret;
}
@@ -276,6 +281,7 @@ public class CliDriver {
}
qp.close();
+ ShimLoader.getHadoopShims().setHadoopSessionContext(ss.getSessionId());
if (out instanceof FetchConverter) {
((FetchConverter) out).fetchFinished();
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java
----------------------------------------------------------------------
diff --git a/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java b/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java
index 8f8f04e..8419c33 100644
--- a/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java
+++ b/cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.ql.IDriver;
+import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.junit.Test;
@@ -165,8 +166,10 @@ public class TestCliDriverMethods extends TestCase {
IDriver proc = mock(IDriver.class);
CommandProcessorResponse cpr = mock(CommandProcessorResponse.class);
+ QueryState queryState = new QueryState.Builder().withGenerateNewQueryId(true).build();
when(cpr.getResponseCode()).thenReturn(0);
when(proc.run(anyString())).thenReturn(cpr);
+ when(proc.getQueryState()).thenReturn(queryState);
// and then see what happens based on the provided schema
when(proc.getSchema()).thenReturn(mockSchema);
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 6441e67..6990be0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -2975,6 +2975,7 @@ public class Driver implements IDriver {
this.operationId = opId;
}
+ @Override
public QueryState getQueryState() {
return queryState;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/ql/src/java/org/apache/hadoop/hive/ql/IDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/IDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/IDriver.java
index bab5014..e44e6a3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/IDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/IDriver.java
@@ -42,6 +42,8 @@ public interface IDriver extends CommandProcessor {
QueryPlan getPlan();
+ QueryState getQueryState();
+
QueryDisplay getQueryDisplay();
void setOperationId(String guid64);
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
index 3bc3b29..ab5c66b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java
@@ -132,6 +132,11 @@ public class ReExecDriver implements IDriver {
}
@Override
+ public QueryState getQueryState() {
+ return queryState;
+ }
+
+ @Override
public QueryDisplay getQueryDisplay() {
return coreDriver.getQueryDisplay();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 9291e11..ee7c940 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -423,6 +423,8 @@ public class SessionState {
resourceDownloader = new ResourceDownloader(conf,
HiveConf.getVar(conf, ConfVars.DOWNLOADED_RESOURCES_DIR));
killQuery = new NullKillQuery();
+
+ ShimLoader.getHadoopShims().setHadoopSessionContext(getSessionId());
}
public Map<String, String> getHiveVariables() {
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/service/src/java/org/apache/hive/service/cli/operation/Operation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
index 4b9cbd3..cb995de 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.log.LogDivertAppender;
import org.apache.hadoop.hive.ql.log.LogDivertAppenderForTest;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.OperationLog;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hive.service.cli.FetchOrientation;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.OperationHandle;
@@ -223,6 +224,7 @@ public abstract class Operation {
* Set up some preconditions, or configurations.
*/
protected void beforeRun() {
+ ShimLoader.getHadoopShims().setHadoopQueryContext(queryState.getQueryId());
createOperationLog();
LogUtils.registerLoggingContext(queryState.getConf());
}
@@ -233,6 +235,8 @@ public abstract class Operation {
*/
protected void afterRun() {
LogUtils.unregisterLoggingContext();
+ // Reset back to session context after the query is done
+ ShimLoader.getHadoopShims().setHadoopSessionContext(parentSession.getSessionState().getSessionId());
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
index 01c9249..3d24884 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.security.UserGroupInformation;
@@ -314,6 +315,8 @@ public class SQLOperation extends ExecuteStatementOperation {
SessionState.setCurrentSessionState(parentSessionState);
PerfLogger.setPerfLogger(SessionState.getPerfLogger());
LogUtils.registerLoggingContext(queryState.getConf());
+ ShimLoader.getHadoopShims().setHadoopQueryContext(queryState.getQueryId());
+
try {
if (asyncPrepare) {
prepare(queryState);
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 98c3eef..b6f70eb 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@ -481,6 +482,21 @@ public class Hadoop23Shims extends HadoopShimsSecure {
return new MiniSparkShim(conf, numberOfTaskTrackers, nameNode, numDir);
}
+ @Override
+ public void setHadoopCallerContext(String callerContext) {
+ CallerContext.setCurrent(new CallerContext.Builder(callerContext).build());
+ }
+
+ @Override
+ public void setHadoopQueryContext(String queryId) {
+ setHadoopCallerContext("hive_queryId_" + queryId);
+ }
+
+ @Override
+ public void setHadoopSessionContext(String sessionId) {
+ setHadoopCallerContext("hive_sessionId_" + sessionId);
+ }
+
/**
* Shim for MiniSparkOnYARNCluster
*/
@@ -1048,7 +1064,6 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
}
-
public static class StoragePolicyShim implements HadoopShims.StoragePolicyShim {
private final DistributedFileSystem dfs;
http://git-wip-us.apache.org/repos/asf/hive/blob/8f3b312e/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 84e6430..c569b24 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -96,6 +96,22 @@ public interface HadoopShims {
public MiniMrShim getMiniSparkCluster(Configuration conf, int numberOfTaskTrackers,
String nameNode, int numDir) throws IOException;
+
+ /**
+ * Set up the caller context for HDFS and Yarn.
+ */
+ void setHadoopCallerContext(String callerContext);
+
+ /**
+ * Set up context specific caller context with query prefix.
+ */
+ void setHadoopQueryContext(String queryId);
+
+ /**
+ * Set up context specific caller context with session prefix.
+ */
+ void setHadoopSessionContext(String sessionId);
+
/**
* Shim for MiniMrCluster
*/