You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ai...@apache.org on 2018/04/30 23:36:02 UTC
hive git commit: HIVE-18915: Better client logging when a HoS session
can't be opened (Aihua Xu, reviewed by Sahil Takiar)
Repository: hive
Updated Branches:
refs/heads/master f8bc4868e -> 3c4a9c6bc
HIVE-18915: Better client logging when a HoS session can't be opened (Aihua Xu, reviewed by Sahil Takiar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c4a9c6b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c4a9c6b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c4a9c6b
Branch: refs/heads/master
Commit: 3c4a9c6bc37dfc870800523a6d26f667d0b43818
Parents: f8bc486
Author: Aihua Xu <ai...@apache.org>
Authored: Thu Apr 12 17:12:56 2018 -0700
Committer: Aihua Xu <ai...@apache.org>
Committed: Mon Apr 30 16:35:41 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 2 +-
.../ql/exec/spark/session/SparkSessionImpl.java | 30 +++++++++-----------
.../session/TestSparkSessionManagerImpl.java | 6 ++--
3 files changed, 17 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3c4a9c6b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 7d33fa3..94dd636 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -576,7 +576,7 @@ public enum ErrorMsg {
SPARK_CREATE_CLIENT_INTERRUPTED(30040,
"Interrupted while creating Spark client for session {0}", true),
SPARK_CREATE_CLIENT_ERROR(30041,
- "Failed to create Spark client for Spark session {0}", true),
+ "Failed to create Spark client for Spark session {0}: {1}", true),
SPARK_CREATE_CLIENT_INVALID_RESOURCE_REQUEST(30042,
"Failed to create Spark client due to invalid resource request: {0}", true),
SPARK_CREATE_CLIENT_CLOSED_SESSION(30043,
http://git-wip-us.apache.org/repos/asf/hive/blob/3c4a9c6b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
index 2d5d03e..189de19 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java
@@ -71,7 +71,6 @@ public class SparkSessionImpl implements SparkSession {
private HiveSparkClient hiveSparkClient;
private Path scratchDir;
private final Object dirLock = new Object();
- private String matchedString = null;
public SparkSessionImpl() {
sessionId = makeSessionId();
@@ -195,6 +194,7 @@ public class SparkSessionImpl implements SparkSession {
@VisibleForTesting
HiveException getHiveException(Throwable e) {
Throwable oe = e;
+ StringBuilder matchedString = new StringBuilder();
while (e != null) {
if (e instanceof TimeoutException) {
return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_TIMEOUT);
@@ -202,31 +202,26 @@ public class SparkSessionImpl implements SparkSession {
return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INTERRUPTED, sessionId);
} else if (e instanceof RuntimeException) {
String sts = Throwables.getStackTraceAsString(e);
- if (matches(sts, AM_TIMEOUT_ERR)) {
+ if (matches(sts, AM_TIMEOUT_ERR, matchedString)) {
return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_TIMEOUT);
- } else if (matches(sts, UNKNOWN_QUEUE_ERR) || matches(sts, STOPPED_QUEUE_ERR)) {
- return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_QUEUE, matchedString);
- } else if (matches(sts, FULL_QUEUE_ERR)) {
- return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_QUEUE_FULL, matchedString);
- } else if (matches(sts, INVALILD_MEM_ERR) || matches(sts, INVALID_CORE_ERR)) {
+ } else if (matches(sts, UNKNOWN_QUEUE_ERR, matchedString) || matches(sts, STOPPED_QUEUE_ERR, matchedString)) {
+ return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_QUEUE, matchedString.toString());
+ } else if (matches(sts, FULL_QUEUE_ERR, matchedString)) {
+ return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_QUEUE_FULL, matchedString.toString());
+ } else if (matches(sts, INVALILD_MEM_ERR, matchedString) || matches(sts, INVALID_CORE_ERR, matchedString)) {
return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_RESOURCE_REQUEST,
- matchedString);
+ matchedString.toString());
} else {
- return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId);
+ return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId, Throwables.getRootCause(e).getMessage());
}
}
e = e.getCause();
}
- return new HiveException(oe, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId);
+ return new HiveException(oe, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId, Throwables.getRootCause(oe).getMessage());
}
- @VisibleForTesting
- String getMatchedString() {
- return matchedString;
- }
-
- private boolean matches(String input, String regex) {
+ private boolean matches(String input, String regex, StringBuilder matchedString) {
if (!errorPatterns.containsKey(regex)) {
LOG.warn("No error pattern found for regex: {}", regex);
return false;
@@ -235,7 +230,8 @@ public class SparkSessionImpl implements SparkSession {
Matcher m = p.matcher(input);
boolean result = m.find();
if (result && m.groupCount() == 1) {
- this.matchedString = m.group(1);
+ // assume matchedString is empty
+ matchedString.append(m.group(1));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3c4a9c6b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
index fe95ce0..5924b8b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
@@ -183,8 +183,8 @@ public class TestSparkSessionManagerImpl {
"initial executor number 5 must between min executor number10 and max executor number 50");
// Other exceptions which defaults to SPARK_CREATE_CLIENT_ERROR
- e = new Exception();
- checkHiveException(ss, e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR);
+ e = new Exception("Other exception");
+ checkHiveException(ss, e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, "Other exception");
}
private void checkHiveException(SparkSessionImpl ss, Throwable e, ErrorMsg expectedErrMsg) {
@@ -196,7 +196,7 @@ public class TestSparkSessionManagerImpl {
HiveException he = ss.getHiveException(e);
assertEquals(expectedErrMsg, he.getCanonicalErrorMsg());
if (expectedMatchedStr != null) {
- assertEquals(expectedMatchedStr, ss.getMatchedString());
+ assertTrue(he.getMessage().contains(expectedMatchedStr));
}
}