You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2016/01/29 19:37:42 UTC
hive git commit: HIVE-12550 : Cache and display last N completed
queries in HS2 WebUI (Szehon, reviewed by Aihua Xu and Mohit Sabharwal)
Repository: hive
Updated Branches:
refs/heads/master 41fc1874b -> 3db7227fc
HIVE-12550 : Cache and display last N completed queries in HS2 WebUI (Szehon, reviewed by Aihua Xu and Mohit Sabharwal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3db7227f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3db7227f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3db7227f
Branch: refs/heads/master
Commit: 3db7227fc7e40ef65d59a8e5d26b9c83bc6fe08c
Parents: 41fc187
Author: Szehon Ho <sz...@cloudera.com>
Authored: Fri Jan 22 15:37:37 2016 -0500
Committer: Szehon Ho <sz...@cloudera.com>
Committed: Fri Jan 29 10:36:34 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
pom.xml | 2 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 9 ++++
.../hive/service/cli/operation/Operation.java | 12 ++++-
.../service/cli/operation/OperationManager.java | 49 +++++++++++++++++++-
.../service/cli/operation/SQLOperation.java | 22 ++++++++-
.../service/cli/operation/SQLOperationInfo.java | 48 +++++++++++++++++++
.../hive-webapps/hiveserver2/hiveserver2.jsp | 41 +++++++++++++++-
8 files changed, 180 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index bfd88f8..a78f78a 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1910,6 +1910,8 @@ public class HiveConf extends Configuration {
"HTTP/_HOST@EXAMPLE.COM", "The HiveServer2 WebUI SPNEGO service principal.\n" +
"The special string _HOST will be replaced automatically with \n" +
"the value of hive.server2.webui.host or the correct host name."),
+ HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES("hive.server2.webui.max.historic.queries", 25,
+ "The maximum number of past queries to show in HiverSever2 WebUI."),
// Tez session settings
HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "",
http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2d2a3de..802d3d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -125,7 +125,7 @@
<commons-dbcp.version>1.4</commons-dbcp.version>
<derby.version>10.10.2.0</derby.version>
<dropwizard.version>3.1.0</dropwizard.version>
- <guava.version>14.0.1</guava.version>
+ <guava.version>15.0</guava.version>
<groovy.version>2.4.4</groovy.version>
<hadoop.version>2.6.0</hadoop.version>
<hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path>
http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 75187cf..4c89812 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -158,6 +158,9 @@ public class Driver implements CommandProcessor {
// HS2 operation handle guid string
private String operationId;
+ // For WebUI. Kept alive after queryPlan is freed.
+ private String savedQueryString;
+
private boolean checkConcurrency() {
boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY);
if (!supportConcurrency) {
@@ -384,6 +387,7 @@ public class Driver implements CommandProcessor {
} catch (Exception e) {
LOG.warn("WARNING! Query command could not be redacted." + e);
}
+ this.savedQueryString = queryStr;
//holder for parent command type/string when executing reentrant queries
QueryState queryState = new QueryState();
@@ -1943,6 +1947,11 @@ public class Driver implements CommandProcessor {
return errorMessage;
}
+
+ public String getQueryString() {
+ return savedQueryString == null ? "Unknown" : savedQueryString;
+ }
+
/**
* Set the HS2 operation handle's guid string
* @param opId base64 encoded guid string
http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/Operation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
index d6ac16d..0c263cf 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
@@ -75,6 +75,7 @@ public abstract class Operation {
private long operationTimeout;
private volatile long lastAccessTime;
+ private final long beginTime;
protected static final EnumSet<FetchOrientation> DEFAULT_FETCH_ORIENTATION_SET =
EnumSet.of(FetchOrientation.FETCH_NEXT,FetchOrientation.FETCH_FIRST);
@@ -92,7 +93,8 @@ public abstract class Operation {
}
this.runAsync = runInBackground;
this.opHandle = new OperationHandle(opType, parentSession.getProtocolVersion());
- lastAccessTime = System.currentTimeMillis();
+ beginTime = System.currentTimeMillis();
+ lastAccessTime = beginTime;
operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(),
HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS);
setMetrics(state);
@@ -410,4 +412,12 @@ public abstract class Operation {
}
}
}
+
+ public long getBeginTime() {
+ return beginTime;
+ }
+
+ protected OperationState getState() {
+ return state;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
index 92135cd..f1ce6f6 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
@@ -22,10 +22,14 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.collect.EvictingQueue;
import org.apache.hadoop.hive.common.metrics.common.Metrics;
import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
@@ -61,6 +65,9 @@ public class OperationManager extends AbstractService {
private final ConcurrentHashMap<OperationHandle, Operation> handleToOperation =
new ConcurrentHashMap<OperationHandle, Operation>();
+ //for displaying historical queries on WebUI
+ private Queue<SQLOperationInfo> historicSqlOperations;
+
public OperationManager() {
super(OperationManager.class.getSimpleName());
}
@@ -73,6 +80,11 @@ public class OperationManager extends AbstractService {
} else {
LOG.debug("Operation level logging is turned off");
}
+ if ((hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) &&
+ hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) {
+ historicSqlOperations = EvictingQueue.create(
+ hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES));
+ }
super.init(hiveConf);
}
@@ -174,6 +186,7 @@ public class OperationManager extends AbstractService {
Operation operation = handleToOperation.get(operationHandle);
if (operation != null && operation.isTimedOut(System.currentTimeMillis())) {
handleToOperation.remove(operationHandle, operation);
+ cacheOldOperationInfo(operation);
return operation;
}
return null;
@@ -184,7 +197,9 @@ public class OperationManager extends AbstractService {
}
private Operation removeOperation(OperationHandle opHandle) {
- return handleToOperation.remove(opHandle);
+ Operation result = handleToOperation.remove(opHandle);
+ cacheOldOperationInfo(result);
+ return result;
}
public OperationStatus getOperationStatus(OperationHandle opHandle)
@@ -313,4 +328,36 @@ public class OperationManager extends AbstractService {
}
return removed;
}
+
+ //Cache a number of historical operation info, at max number of
+ //hive.server2.webui.max.historic.queries.
+ private void cacheOldOperationInfo(Operation oldOperation) {
+ if ((getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) &&
+ getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) {
+ if (oldOperation instanceof SQLOperation) {
+ SQLOperation query = (SQLOperation) oldOperation;
+ SQLOperationInfo queryInfo = query.getSQLOperationInfo();
+ if (queryInfo != null) {
+ synchronized (historicSqlOperations) {
+ historicSqlOperations.add(queryInfo);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * @return a number of historical SQLOperation info, at max number of
+ * hive.server2.webui.max.historic.queries
+ */
+ public List<SQLOperationInfo> getHistoricalSQLOpInfo() {
+ List<SQLOperationInfo> result = new LinkedList<>();
+ synchronized (historicSqlOperations) {
+ Iterator<SQLOperationInfo> opIterator = historicSqlOperations.iterator();
+ while (opIterator.hasNext()) {
+ result.add(opIterator.next());
+ }
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
index c8a69b9..01b1d3d 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -163,7 +163,7 @@ public class SQLOperation extends ExecuteStatementOperation {
}
public String getQueryStr() {
- return driver == null || driver.getPlan() == null ? "Unknown" : driver.getPlan().getQueryStr();
+ return driver == null ? "Unknown" : driver.getQueryString();
}
private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException {
@@ -199,6 +199,7 @@ public class SQLOperation extends ExecuteStatementOperation {
public void runInternal() throws HiveSQLException {
setState(OperationState.PENDING);
final HiveConf opConfig = getConfigForOperation();
+
prepare(opConfig);
if (!shouldRunAsync()) {
runQuery(opConfig);
@@ -304,6 +305,7 @@ public class SQLOperation extends ExecuteStatementOperation {
backgroundHandle.cancel(true);
}
}
+
if (driver != null) {
driver.close();
driver.destroy();
@@ -479,4 +481,22 @@ public class SQLOperation extends ExecuteStatementOperation {
}
return sqlOperationConf;
}
+
+ /**
+ * Get summary information of this SQLOperation for display in WebUI.
+ */
+ public SQLOperationInfo getSQLOperationInfo() {
+ try {
+ return new SQLOperationInfo(
+ getParentSession().getUserName(),
+ driver.getQueryString(),
+ getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE),
+ getState(),
+ (int) (System.currentTimeMillis() - getBeginTime()) / 1000,
+ System.currentTimeMillis());
+ } catch (HiveSQLException e) {
+ LOG.warn("Error calcluating SQL Operation Info for webui", e);
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
new file mode 100644
index 0000000..179f6dd
--- /dev/null
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.service.cli.operation;
+
+import org.apache.hive.service.cli.OperationState;
+
+/**
+ * Used to display some info in the HS2 WebUI.
+ */
+public class SQLOperationInfo {
+ public String userName;
+ public String queryStr;
+ public String executionEngine;
+ public OperationState endState; //state before CLOSED (one of CANCELLED, FINISHED, ERROR)
+ public int elapsedTime;
+ public long endTime;
+
+ public SQLOperationInfo(
+ String userName,
+ String queryStr,
+ String executionEngine,
+ OperationState endState,
+ int elapsedTime,
+ long endTime
+ ) {
+ this.userName = userName;
+ this.queryStr = queryStr;
+ this.executionEngine = executionEngine;
+ this.endState = endState;
+ this.elapsedTime = elapsedTime;
+ this.endTime = endTime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/3db7227f/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
----------------------------------------------------------------------
diff --git a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
index a91b008..a0b5d2e 100644
--- a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
+++ b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp
@@ -24,11 +24,13 @@
import="org.apache.hive.common.util.HiveVersionInfo"
import="org.apache.hive.service.cli.operation.Operation"
import="org.apache.hive.service.cli.operation.SQLOperation"
+ import="org.apache.hive.service.cli.operation.SQLOperationInfo"
import="org.apache.hive.service.cli.session.SessionManager"
import="org.apache.hive.service.cli.session.HiveSession"
import="javax.servlet.ServletContext"
import="java.util.Collection"
import="java.util.Date"
+ import="java.util.List"
%>
<%
@@ -145,7 +147,7 @@ for (Operation operation: operations) {
<td><%= query.getQueryStr() %></td>
<td><%= query.getConfigForOperation().getVar(ConfVars.HIVE_EXECUTION_ENGINE) %>
<td><%= query.getStatus().getState() %></td>
- <td><%= (currentTime - query.getLastAccessTime())/1000 %></td>
+ <td><%= (currentTime - query.getBeginTime())/1000 %></td>
</tr>
<%
}
@@ -156,6 +158,43 @@ for (Operation operation: operations) {
</tr>
</table>
</section>
+
+
+<section>
+<h2>Last Max <%= conf.get(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES.varname) %> Completed Queries</h2>
+<table id="attributes_table" class="table table-striped">
+ <tr>
+ <th>User Name</th>
+ <th>Query</th>
+ <th>Execution Engine</th>
+ <th>State</th>
+ <th>Elapsed Time (s)</th>
+ <th>End Time </th>
+ </tr>
+<%
+queries = 0;
+List<SQLOperationInfo> sqlOperations = sessionManager.getOperationManager().getHistoricalSQLOpInfo();
+for (SQLOperationInfo sqlOperation: sqlOperations) {
+ queries++;
+%>
+ <tr>
+ <td><%= sqlOperation.userName %></td>
+ <td><%= sqlOperation.queryStr %></td>
+ <td><%= sqlOperation.executionEngine %></td>
+ <td><%= sqlOperation.endState %></td>
+ <td><%= sqlOperation.elapsedTime %></td>
+ <td><%= new Date(sqlOperation.endTime) %></td>
+ </tr>
+<%
+}
+
+%>
+<tr>
+ <td colspan="6">Total number of queries: <%= queries %></td>
+</tr>
+</table>
+</section>
+
<%
}
%>