You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2017/10/30 08:40:54 UTC
hive git commit: HIVE-17888: Display the reason for query
cancellation (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master d6dd543b2 -> fce8a4502
HIVE-17888: Display the reason for query cancellation (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fce8a450
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fce8a450
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fce8a450
Branch: refs/heads/master
Commit: fce8a45028b34c2763997327f107992245bb4fc8
Parents: d6dd543
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Mon Oct 30 01:40:38 2017 -0700
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Mon Oct 30 01:40:38 2017 -0700
----------------------------------------------------------------------
.../jdbc/TestTriggersTezSessionPoolManager.java | 16 ++++++-------
.../org/apache/hive/jdbc/HiveStatement.java | 8 ++++++-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 2 +-
.../ql/exec/tez/KillTriggerActionHandler.java | 2 +-
.../ql/exec/tez/TriggerValidatorRunnable.java | 5 +++-
.../exec/tez/TriggerViolationActionHandler.java | 2 +-
.../hadoop/hive/ql/session/KillQuery.java | 2 +-
.../hadoop/hive/ql/session/NullKillQuery.java | 2 +-
.../org/apache/hadoop/hive/ql/wm/Trigger.java | 9 +++++++
.../apache/hive/service/cli/OperationState.java | 9 +++++++
.../service/cli/operation/OperationManager.java | 25 ++++++++++++--------
.../service/cli/thrift/ThriftCLIService.java | 1 +
.../hive/service/server/KillQueryImpl.java | 4 ++--
13 files changed, 60 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java
index 8d962b5..c195243 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java
@@ -120,7 +120,7 @@ public class TestTriggersTezSessionPoolManager {
setupTriggers(Lists.newArrayList(trigger));
String query = "select sleep(t1.under_col, 500), t1.value from " + tableName + " t1 join " + tableName +
" t2 on t1.under_col>=t2.under_col";
- runQueryWithTrigger(query, null, "Query was cancelled");
+ runQueryWithTrigger(query, null, trigger + " violated");
}
@Test(timeout = 60000)
@@ -130,7 +130,7 @@ public class TestTriggersTezSessionPoolManager {
setupTriggers(Lists.newArrayList(trigger));
String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName +
" t2 on t1.under_col>=t2.under_col";
- runQueryWithTrigger(query, null, "Query was cancelled");
+ runQueryWithTrigger(query, null, trigger + " violated");
}
@Test(timeout = 60000)
@@ -148,7 +148,7 @@ public class TestTriggersTezSessionPoolManager {
setCmds.add("set tez.grouping.max-size=400");
String query = "select t1.under_col, t1.value from " + tableName + " t1 join " + tableName +
" t2 on t1.under_col>=t2.under_col order by t1.under_col";
- runQueryWithTrigger(query, setCmds, "Query was cancelled");
+ runQueryWithTrigger(query, setCmds, trigger + " violated");
}
@Test(timeout = 60000)
@@ -158,7 +158,7 @@ public class TestTriggersTezSessionPoolManager {
setupTriggers(Lists.newArrayList(trigger));
String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName +
" t2 on t1.under_col>=t2.under_col";
- runQueryWithTrigger(query, null, "Query was cancelled");
+ runQueryWithTrigger(query, null, trigger + " violated");
}
@Test(timeout = 60000)
@@ -168,7 +168,7 @@ public class TestTriggersTezSessionPoolManager {
setupTriggers(Lists.newArrayList(trigger));
String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName +
" t2 on t1.under_col>=t2.under_col";
- runQueryWithTrigger(query, null, "Query was cancelled");
+ runQueryWithTrigger(query, null, trigger + " violated");
}
@Test(timeout = 60000)
@@ -178,7 +178,7 @@ public class TestTriggersTezSessionPoolManager {
setupTriggers(Lists.newArrayList(trigger));
String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName +
" t2 on t1.under_col>=t2.under_col";
- runQueryWithTrigger(query, getConfigs(), "Query was cancelled");
+ runQueryWithTrigger(query, getConfigs(), trigger + " violated");
}
@Test(timeout = 60000)
@@ -309,7 +309,7 @@ public class TestTriggersTezSessionPoolManager {
setupTriggers(Lists.newArrayList(shuffleTrigger, execTimeTrigger));
String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName +
" t2 on t1.under_col>=t2.under_col";
- runQueryWithTrigger(query, null, "Query was cancelled");
+ runQueryWithTrigger(query, null, execTimeTrigger + " violated");
}
@Test(timeout = 60000)
@@ -321,7 +321,7 @@ public class TestTriggersTezSessionPoolManager {
setupTriggers(Lists.newArrayList(shuffleTrigger, execTimeTrigger));
String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName +
" t2 on t1.under_col>=t2.under_col";
- runQueryWithTrigger(query, null, "Query was cancelled");
+ runQueryWithTrigger(query, null, shuffleTrigger + " violated");
}
private void createSleepUDF() throws SQLException {
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
index a8e3bd9..0f2193f 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
@@ -37,6 +37,7 @@ import org.apache.hive.service.rpc.thrift.TGetOperationStatusReq;
import org.apache.hive.service.rpc.thrift.TGetOperationStatusResp;
import org.apache.hive.service.rpc.thrift.TGetQueryIdReq;
import org.apache.hive.service.rpc.thrift.TOperationHandle;
+import org.apache.hive.service.rpc.thrift.TOperationState;
import org.apache.hive.service.rpc.thrift.TSessionHandle;
import org.apache.thrift.TException;
import org.slf4j.Logger;
@@ -386,7 +387,12 @@ public class HiveStatement implements java.sql.Statement {
break;
case CANCELED_STATE:
// 01000 -> warning
- throw new SQLException("Query was cancelled", "01000");
+ String errMsg = statusResp.getErrorMessage();
+ if (errMsg != null && !errMsg.isEmpty()) {
+ throw new SQLException("Query was cancelled. " + errMsg, "01000");
+ } else {
+ throw new SQLException("Query was cancelled", "01000");
+ }
case TIMEDOUT_STATE:
throw new SQLTimeoutException("Query timed out after " + queryTimeout + " seconds");
case ERROR_STATE:
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 87f6e1d..e33c7ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -3136,7 +3136,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
private int killQuery(Hive db, KillQueryDesc desc) throws HiveException {
SessionState sessionState = SessionState.get();
for (String queryId : desc.getQueryIds()) {
- sessionState.getKillQuery().killQuery(queryId);
+ sessionState.getKillQuery().killQuery(queryId, "User invoked KILL QUERY");
}
LOG.info("kill query called ({})", desc.getQueryIds());
return 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
index 4c15d90..474fae9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
@@ -42,7 +42,7 @@ public class KillTriggerActionHandler implements TriggerActionHandler {
KillQuery killQuery = sessionState.getKillQuery();
// if kill query is null then session might have been released to pool or closed already
if (killQuery != null) {
- sessionState.getKillQuery().killQuery(queryId);
+ sessionState.getKillQuery().killQuery(queryId, entry.getValue().getMsg());
}
} catch (HiveException e) {
LOG.warn("Unable to kill query {} for trigger violation");
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerValidatorRunnable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerValidatorRunnable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerValidatorRunnable.java
index b0378eb..33bccbe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerValidatorRunnable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerValidatorRunnable.java
@@ -57,7 +57,10 @@ public class TriggerValidatorRunnable implements Runnable {
String queryId = s.getTriggerContext().getQueryId();
LOG.info("Query {} violated trigger {}. Current counter value: {}. Going to apply action {}", queryId,
t, currentCounterValue, t.getAction());
- violatedSessions.put(s, t.getAction());
+ Trigger.Action action = t.getAction();
+ String msg = "Trigger " + t + " violated. Current value: " + currentCounterValue;
+ action.setMsg(msg);
+ violatedSessions.put(s, action);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerViolationActionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerViolationActionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerViolationActionHandler.java
index ce9147f..c46569b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerViolationActionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerViolationActionHandler.java
@@ -35,7 +35,7 @@ public class TriggerViolationActionHandler implements TriggerActionHandler {
TezSessionState sessionState = entry.getKey();
String queryId = sessionState.getTriggerContext().getQueryId();
try {
- sessionState.getKillQuery().killQuery(queryId);
+ sessionState.getKillQuery().killQuery(queryId, entry.getValue().getMsg());
} catch (HiveException e) {
LOG.warn("Unable to kill query {} for trigger violation");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/ql/src/java/org/apache/hadoop/hive/ql/session/KillQuery.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/KillQuery.java b/ql/src/java/org/apache/hadoop/hive/ql/session/KillQuery.java
index 4bc81b3..a110ce7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/KillQuery.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/KillQuery.java
@@ -21,5 +21,5 @@ package org.apache.hadoop.hive.ql.session;
import org.apache.hadoop.hive.ql.metadata.HiveException;
public interface KillQuery {
- void killQuery(String queryId) throws HiveException;
+ void killQuery(String queryId, String errMsg) throws HiveException;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/ql/src/java/org/apache/hadoop/hive/ql/session/NullKillQuery.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/NullKillQuery.java b/ql/src/java/org/apache/hadoop/hive/ql/session/NullKillQuery.java
index eab8fbf..ea633ba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/NullKillQuery.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/NullKillQuery.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
public class NullKillQuery implements KillQuery {
@Override
- public void killQuery(String queryId) throws HiveException {
+ public void killQuery(String queryId, String errMsg) throws HiveException {
// Do nothing
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/ql/src/java/org/apache/hadoop/hive/ql/wm/Trigger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/wm/Trigger.java b/ql/src/java/org/apache/hadoop/hive/ql/wm/Trigger.java
index bed0ac1..6299f2b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/wm/Trigger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/wm/Trigger.java
@@ -27,6 +27,7 @@ public interface Trigger {
MOVE_TO_POOL("");
String poolName;
+ String msg;
Action(final String poolName) {
this.poolName = poolName;
@@ -40,6 +41,14 @@ public interface Trigger {
public String getPoolName() {
return poolName;
}
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public void setMsg(final String msg) {
+ this.msg = msg;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/service/src/java/org/apache/hive/service/cli/OperationState.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/OperationState.java b/service/src/java/org/apache/hive/service/cli/OperationState.java
index ae1ff5e..cda319a 100644
--- a/service/src/java/org/apache/hive/service/cli/OperationState.java
+++ b/service/src/java/org/apache/hive/service/cli/OperationState.java
@@ -37,6 +37,7 @@ public enum OperationState {
private final TOperationState tOperationState;
private final boolean terminal;
+ private String errorMessage;
OperationState(TOperationState tOperationState, boolean terminal) {
this.tOperationState = tOperationState;
@@ -109,4 +110,12 @@ public enum OperationState {
public boolean isTerminal() {
return terminal;
}
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+ public void setErrorMessage(final String errorMessage) {
+ this.errorMessage = errorMessage;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
index 8224bcc..257ab54 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hive.ql.QueryInfo;
import org.apache.hadoop.hive.ql.log.LogDivertAppender;
import org.apache.hadoop.hive.ql.log.LogDivertAppenderForTest;
import org.apache.hadoop.hive.ql.session.OperationLog;
-import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.AbstractService;
import org.apache.hive.service.cli.FetchOrientation;
import org.apache.hive.service.cli.HiveSQLException;
@@ -50,12 +49,6 @@ import org.apache.hive.service.cli.RowSet;
import org.apache.hive.service.cli.RowSetFactory;
import org.apache.hive.service.cli.TableSchema;
import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.hive.service.server.KillQueryImpl;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.core.Appender;
-import org.apache.logging.log4j.core.LoggerContext;
-import org.apache.logging.log4j.core.config.Configuration;
-import org.apache.logging.log4j.core.config.LoggerConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -259,10 +252,11 @@ public class OperationManager extends AbstractService {
/**
* Cancel the running operation unless it is already in a terminal state
- * @param opHandle
+ * @param opHandle operation handle
+ * @param errMsg error message
* @throws HiveSQLException
*/
- public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
+ public void cancelOperation(OperationHandle opHandle, String errMsg) throws HiveSQLException {
Operation operation = getOperation(opHandle);
OperationState opState = operation.getStatus().getState();
if (opState.isTerminal()) {
@@ -270,13 +264,24 @@ public class OperationManager extends AbstractService {
LOG.debug(opHandle + ": Operation is already aborted in state - " + opState);
} else {
LOG.debug(opHandle + ": Attempting to cancel from state - " + opState);
- operation.cancel(OperationState.CANCELED);
+ OperationState operationState = OperationState.CANCELED;
+ operationState.setErrorMessage(errMsg);
+ operation.cancel(operationState);
if (operation instanceof SQLOperation) {
removeSafeQueryInfo(opHandle);
}
}
}
+ /**
+ * Cancel the running operation unless it is already in a terminal state
+ * @param opHandle
+ * @throws HiveSQLException
+ */
+ public void cancelOperation(OperationHandle opHandle) throws HiveSQLException {
+ cancelOperation(opHandle, "");
+ }
+
public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
LOG.info("Closing operation: " + opHandle);
Operation operation = removeOperation(opHandle);
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index 7012a25..6354c8c 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -661,6 +661,7 @@ public abstract class ThriftCLIService extends AbstractService implements TCLISe
OperationStatus operationStatus =
cliService.getOperationStatus(operationHandle, req.isGetProgressUpdate());
resp.setOperationState(operationStatus.getState().toTOperationState());
+ resp.setErrorMessage(operationStatus.getState().getErrorMessage());
HiveSQLException opException = operationStatus.getOperationException();
resp.setTaskStatus(operationStatus.getTaskStatus());
resp.setOperationStarted(operationStatus.getOperationStarted());
http://git-wip-us.apache.org/repos/asf/hive/blob/fce8a450/service/src/java/org/apache/hive/service/server/KillQueryImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/server/KillQueryImpl.java b/service/src/java/org/apache/hive/service/server/KillQueryImpl.java
index 5412371..f302b4b 100644
--- a/service/src/java/org/apache/hive/service/server/KillQueryImpl.java
+++ b/service/src/java/org/apache/hive/service/server/KillQueryImpl.java
@@ -37,14 +37,14 @@ public class KillQueryImpl implements KillQuery {
}
@Override
- public void killQuery(String queryId) throws HiveException {
+ public void killQuery(String queryId, String errMsg) throws HiveException {
try {
Operation operation = operationManager.getOperationByQueryId(queryId);
if (operation == null) {
LOG.info("Query not found: " + queryId);
} else {
OperationHandle handle = operation.getHandle();
- operationManager.cancelOperation(handle);
+ operationManager.cancelOperation(handle, errMsg);
}
} catch (HiveSQLException e) {
throw new HiveException(e);