You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by st...@apache.org on 2018/10/18 15:52:20 UTC
hive git commit: HIVE-20488: SparkSubmitSparkClient#launchDriver
should parse exceptions, not just errors (Bharathkrishna Guruvayoor Murali,
reviewed by Sahil Takiar)
Repository: hive
Updated Branches:
refs/heads/master e8b87bfb0 -> d67d52c8d
HIVE-20488: SparkSubmitSparkClient#launchDriver should parse exceptions, not just errors (Bharathkrishna Guruvayoor Murali, reviewed by Sahil Takiar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d67d52c8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d67d52c8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d67d52c8
Branch: refs/heads/master
Commit: d67d52c8d56d41f5d8ca5c605ee10373dafb5bd1
Parents: e8b87bf
Author: Bharathkrishna Guruvayoor Murali <bh...@cloudera.com>
Authored: Thu Oct 18 10:36:02 2018 -0500
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Thu Oct 18 10:36:41 2018 -0500
----------------------------------------------------------------------
.../apache/hive/spark/client/SparkClientUtilities.java | 7 ++++++-
.../apache/hive/spark/client/SparkSubmitSparkClient.java | 5 ++---
.../org/apache/hive/spark/client/TestSparkClient.java | 11 +++++++++++
3 files changed, 19 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d67d52c8/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
----------------------------------------------------------------------
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
index d3cb3dd..b434d8f 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java
@@ -19,6 +19,7 @@
package org.apache.hive.spark.client;
import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.io.File;
@@ -49,7 +50,7 @@ public class SparkClientUtilities {
public static final String HIVE_KRYO_REG_NAME = "org.apache.hive.spark.HiveKryoRegistrator";
private static final String HIVE_KRYO_REG_JAR_NAME = "hive-kryo-registrator";
-
+ private static final ImmutableList<String> ERROR_KEYWORDS = ImmutableList.of("error", "exception");
/**
* Add new elements to the classpath.
*
@@ -186,4 +187,8 @@ public class SparkClientUtilities {
Thread.currentThread().setContextClassLoader(newLoader);
}
}
+
+ public static boolean containsErrorKeyword(String line) {
+ return ERROR_KEYWORDS.stream().anyMatch(x -> StringUtils.containsIgnoreCase(line, x));
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d67d52c8/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java
----------------------------------------------------------------------
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java
index 1879829..f42fee2 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkSubmitSparkClient.java
@@ -34,8 +34,6 @@ import java.util.concurrent.FutureTask;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
-import org.apache.commons.lang3.StringUtils;
-
import org.apache.hadoop.hive.common.log.LogRedirector;
import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -44,6 +42,7 @@ import org.apache.hive.spark.client.rpc.RpcServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hive.spark.client.SparkClientUtilities.containsErrorKeyword;
/**
* Extends the {@link AbstractSparkClient} and launches a child process to run Spark's {@code
@@ -228,7 +227,7 @@ class SparkSubmitSparkClient extends AbstractSparkClient {
List<String> errorMessages = new ArrayList<>();
synchronized (childErrorLog) {
for (String line : childErrorLog) {
- if (StringUtils.containsIgnoreCase(line, "Error")) {
+ if (containsErrorKeyword(line)) {
errorMessages.add("\"" + line + "\"");
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d67d52c8/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
----------------------------------------------------------------------
diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
index 996b24e..0e1557e 100644
--- a/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
+++ b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
@@ -301,6 +301,17 @@ public class TestSparkClient {
});
}
+ @Test
+ public void testErrorParsing() {
+ assertTrue(SparkClientUtilities.containsErrorKeyword("Error.. Test"));
+ assertTrue(SparkClientUtilities.containsErrorKeyword("This line has error.."));
+ assertTrue(SparkClientUtilities.containsErrorKeyword("Test that line has ExcePtion.."));
+ assertTrue(SparkClientUtilities.containsErrorKeyword("Here is eRRor in line.."));
+ assertTrue(SparkClientUtilities.containsErrorKeyword("Here is ExceptioNn in line.."));
+ assertTrue(SparkClientUtilities.containsErrorKeyword("Here is ERROR and Exception in line.."));
+ assertFalse(SparkClientUtilities.containsErrorKeyword("No problems in this line"));
+ }
+
private static final Logger LOG = LoggerFactory.getLogger(TestSparkClient.class);
private <T extends Serializable> JobHandle.Listener<T> newListener() {