You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2015/11/17 21:19:51 UTC
[40/43] hive git commit: HIVE-11488: Add sessionId and queryId info
to HS2 log (Aihua Xu, reviewed by Szehon Ho)
HIVE-11488: Add sessionId and queryId info to HS2 log (Aihua Xu, reviewed by Szehon Ho)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4ca87745
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4ca87745
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4ca87745
Branch: refs/heads/master-fixed
Commit: 4ca877456bc4c591821f7b6cc5a4010305ff7cd4
Parents: 0fee737
Author: aihuaxu <ai...@apache.org>
Authored: Tue Nov 10 15:13:22 2015 -0500
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Nov 17 12:18:35 2015 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 1 +
.../cli/session/TestHiveSessionImpl.java | 1 +
.../java/org/apache/hadoop/hive/ql/Driver.java | 10 ++++++---
.../hive/service/cli/operation/Operation.java | 23 ++++++++++++++++++++
.../service/cli/operation/SQLOperation.java | 2 ++
.../service/cli/session/HiveSessionBase.java | 3 ---
.../service/cli/session/HiveSessionImpl.java | 14 ++++++++++++
7 files changed, 48 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4ca87745/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 425c7d9..838f25c 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3106,6 +3106,7 @@ public class HiveConf extends Configuration {
ConfVars.OUTPUT_FILE_EXTENSION.varname,
ConfVars.SHOW_JOB_FAIL_DEBUG_INFO.varname,
ConfVars.TASKLOG_DEBUG_TIMEOUT.varname,
+ ConfVars.HIVEQUERYID.varname,
};
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/4ca87745/itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestHiveSessionImpl.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestHiveSessionImpl.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestHiveSessionImpl.java
index 2d9ad03..8a32a07 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestHiveSessionImpl.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestHiveSessionImpl.java
@@ -74,6 +74,7 @@ public class TestHiveSessionImpl {
try {
//Running a normal async query with no exceptions,then no need to close opHandle
+ session.open(new HashMap<String, String>());
session.executeStatementAsync(hql, confOverlay);
Mockito.verify(operationManager, Mockito.times(0)).closeOperation(opHandle);
http://git-wip-us.apache.org/repos/asf/hive/blob/4ca87745/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 93c7a54..9a5495b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -384,9 +384,13 @@ public class Driver implements CommandProcessor {
}
saveSession(queryState);
- // generate new query id
- String queryId = QueryPlan.makeQueryId();
- conf.setVar(HiveConf.ConfVars.HIVEQUERYID, queryId);
+ // Generate new query id if it's not set for CLI case. If it's session based,
+ // query id is passed in from the client or initialized when the session starts.
+ String queryId = conf.getVar(HiveConf.ConfVars.HIVEQUERYID);
+ if (queryId == null || queryId.isEmpty()) {
+ queryId = QueryPlan.makeQueryId();
+ conf.setVar(HiveConf.ConfVars.HIVEQUERYID, queryId);
+ }
SessionState.get().setupQueryCurrentTimestamp();
http://git-wip-us.apache.org/repos/asf/hive/blob/4ca87745/service/src/java/org/apache/hive/service/cli/operation/Operation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
index 4ca0561..a851936 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
@@ -31,6 +31,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.OperationLog;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.cli.FetchOrientation;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.OperationHandle;
@@ -41,8 +42,13 @@ import org.apache.hive.service.cli.RowSet;
import org.apache.hive.service.cli.TableSchema;
import org.apache.hive.service.cli.session.HiveSession;
import org.apache.hive.service.cli.thrift.TProtocolVersion;
+import org.apache.logging.log4j.ThreadContext;
public abstract class Operation {
+ // Constants of the key strings for the log4j ThreadContext.
+ private static final String QUERYID = "QueryId";
+ private static final String SESSIONID = "SessionId";
+
protected final HiveSession parentSession;
private OperationState state = OperationState.INITIALIZED;
private final OperationHandle opHandle;
@@ -238,6 +244,22 @@ public abstract class Operation {
*/
protected void beforeRun() {
createOperationLog();
+ registerLoggingContext();
+ }
+
+ /**
+ * Register logging context so that Log4J can print QueryId and/or SessionId for each message
+ */
+ protected void registerLoggingContext() {
+ ThreadContext.put(QUERYID, SessionState.get().getQueryId());
+ ThreadContext.put(SESSIONID, SessionState.get().getSessionId());
+ }
+
+ /**
+ * Unregister logging context
+ */
+ protected void unregisterLoggingContext() {
+ ThreadContext.clearAll();
}
/**
@@ -245,6 +267,7 @@ public abstract class Operation {
* Clean up resources, which was set up in beforeRun().
*/
protected void afterRun() {
+ unregisterLoggingContext();
unregisterOperationLog();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4ca87745/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
index 1c798eb..8b42265 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -225,12 +225,14 @@ public class SQLOperation extends ExecuteStatementOperation {
SessionState.setCurrentSessionState(parentSessionState);
// Set current OperationLog in this async thread for keeping on saving query log.
registerCurrentOperationLog();
+ registerLoggingContext();
try {
runQuery(opConfig);
} catch (HiveSQLException e) {
setOperationException(e);
LOG.error("Error running hive query: ", e);
} finally {
+ unregisterLoggingContext();
unregisterOperationLog();
}
return null;
http://git-wip-us.apache.org/repos/asf/hive/blob/4ca87745/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
index 449d4c7..1ab5652 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java
@@ -18,8 +18,6 @@
package org.apache.hive.service.cli.session;
-import java.util.Map;
-
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.cli.SessionHandle;
@@ -27,7 +25,6 @@ import org.apache.hive.service.cli.operation.OperationManager;
import org.apache.hive.service.cli.thrift.TProtocolVersion;
import java.io.File;
-import java.util.Map;
/**
* Methods that don't need to be executed under a doAs
http://git-wip-us.apache.org/repos/asf/hive/blob/4ca87745/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 27d11df..2d784f0 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -24,6 +24,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -39,6 +40,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.FetchFormatter;
import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -432,6 +434,18 @@ public class HiveSessionImpl implements HiveSession {
throws HiveSQLException {
acquire(true);
+ // Create the queryId if the client doesn't pass in.
+ // Reuse the client's queryId if exists.
+ if (confOverlay == null) {
+ confOverlay = new HashMap<String, String>();
+ }
+ String queryId = confOverlay.get(HiveConf.ConfVars.HIVEQUERYID.varname);
+ if (queryId == null || queryId.isEmpty()) {
+ queryId = QueryPlan.makeQueryId();
+ confOverlay.put(HiveConf.ConfVars.HIVEQUERYID.varname, queryId);
+ sessionState.getConf().setVar(HiveConf.ConfVars.HIVEQUERYID, queryId);
+ }
+
OperationManager operationManager = getOperationManager();
ExecuteStatementOperation operation = operationManager
.newExecuteStatementOperation(getSession(), statement, confOverlay, runAsync);