You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/04/20 00:25:18 UTC
[26/58] [abbrv] hive git commit: HIVE-13183 : Adds MapredLocalTasks
logs in operation log (Rajat Khandelwal, reviwed by Amareshwari)
HIVE-13183 : Adds MapredLocalTasks logs in operation log (Rajat Khandelwal, reviwed by Amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/14bcbab1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/14bcbab1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/14bcbab1
Branch: refs/heads/llap
Commit: 14bcbab13a1dd925943ffa860138e89a0521e888
Parents: dc010a3
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Apr 12 17:20:04 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Apr 12 17:20:04 2016 +0530
----------------------------------------------------------------------
.../apache/hive/common/util/StreamPrinter.java | 18 +++++++++++-------
.../apache/hadoop/hive/ql/exec/TaskRunner.java | 3 +--
.../apache/hadoop/hive/ql/exec/mr/MapRedTask.java | 5 ++---
.../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 9 ++++++---
.../hadoop/hive/ql/session/OperationLog.java | 10 +++++++++-
.../clientpositive/auto_sortmerge_join_8.q.out | 2 ++
.../clientpositive/llap/tez_join_hash.q.out | 4 ++++
.../clientpositive/tez/tez_join_hash.q.out | 4 ++++
8 files changed, 39 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/14bcbab1/common/src/java/org/apache/hive/common/util/StreamPrinter.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/common/util/StreamPrinter.java b/common/src/java/org/apache/hive/common/util/StreamPrinter.java
index 72638fd..1517751 100644
--- a/common/src/java/org/apache/hive/common/util/StreamPrinter.java
+++ b/common/src/java/org/apache/hive/common/util/StreamPrinter.java
@@ -33,12 +33,12 @@ import org.apache.hadoop.io.IOUtils;
public class StreamPrinter extends Thread {
InputStream is;
String type;
- PrintStream os;
+ PrintStream[] outputStreams;
- public StreamPrinter(InputStream is, String type, PrintStream os) {
+ public StreamPrinter(InputStream is, String type, PrintStream... outputStreams) {
this.is = is;
this.type = type;
- this.os = os;
+ this.outputStreams = outputStreams;
}
@Override
@@ -50,18 +50,22 @@ public class StreamPrinter extends Thread {
String line = null;
if (type != null) {
while ((line = br.readLine()) != null) {
- os.println(type + ">" + line);
+ for (PrintStream os: outputStreams) {
+ os.println(type + ">" + line);
+ }
}
} else {
while ((line = br.readLine()) != null) {
- os.println(line);
+ for (PrintStream os: outputStreams) {
+ os.println(line);
+ }
}
}
br.close();
- br=null;
+ br = null;
} catch (IOException ioe) {
ioe.printStackTrace();
- }finally{
+ } finally {
IOUtils.closeStream(br);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/14bcbab1/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
index f6fd081..81f6db0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
**/
public class TaskRunner extends Thread {
-
protected Task<? extends Serializable> tsk;
protected TaskResult result;
protected SessionState ss;
@@ -103,7 +102,7 @@ public class TaskRunner extends Thread {
if (tsk.getException() == null) {
tsk.setException(t);
}
- t.printStackTrace();
+ LOG.error("Error in executeTask", t);
}
result.setExitVal(exitVal, tsk.getException());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/14bcbab1/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
index 310356c..a42c2e9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
@@ -301,8 +301,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
return exitVal;
} catch (Exception e) {
- e.printStackTrace();
- LOG.error("Exception: " + e.getMessage());
+ LOG.error("Got exception", e);
return (1);
} finally {
try {
@@ -313,7 +312,7 @@ public class MapRedTask extends ExecDriver implements Serializable {
}
} catch (Exception e) {
- LOG.error("Exception: " + e.getMessage());
+ LOG.error("Exception: ", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/14bcbab1/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
index f5500a4..c81b14c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.api.StageType;
+import org.apache.hadoop.hive.ql.session.OperationLog;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
@@ -317,8 +318,10 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
CachingPrintStream errPrintStream = new CachingPrintStream(System.err);
- StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out);
- StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, errPrintStream);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out,
+ OperationLog.getCurrentOperationLog().getPrintStream());
+ StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, errPrintStream,
+ OperationLog.getCurrentOperationLog().getPrintStream());
outPrinter.start();
errPrinter.start();
@@ -340,7 +343,7 @@ public class MapredLocalTask extends Task<MapredLocalWork> implements Serializab
return exitVal;
} catch (Exception e) {
- LOG.error("Exception: " + e, e);
+ LOG.error("Exception: ", e);
return (1);
} finally {
if (secureDoAs != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/14bcbab1/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
index 2ecdde9..6d0f14a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java
@@ -39,7 +39,11 @@ public class OperationLog {
private final LogFile logFile;
private LoggingLevel opLoggingLevel = LoggingLevel.UNKNOWN;
- public static enum LoggingLevel {
+ public PrintStream getPrintStream() {
+ return logFile.getPrintStream();
+ }
+
+ public enum LoggingLevel {
NONE, EXECUTION, PERFORMANCE, VERBOSE, UNKNOWN
}
@@ -221,5 +225,9 @@ public class OperationLog {
}
return logs;
}
+
+ public PrintStream getPrintStream() {
+ return out;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/14bcbab1/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out b/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out
index 38996a7..d22ba3b 100644
--- a/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out
+++ b/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out
@@ -1507,6 +1507,8 @@ PREHOOK: Input: default@bucket_small
PREHOOK: Input: default@bucket_small@ds=2008-04-08
PREHOOK: Input: default@bucket_small@ds=2008-04-09
#### A masked pattern was here ####
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask
+ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask
POSTHOOK: query: select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@bucket_big
http://git-wip-us.apache.org/repos/asf/hive/blob/14bcbab1/ql/src/test/results/clientpositive/llap/tez_join_hash.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/tez_join_hash.q.out b/ql/src/test/results/clientpositive/llap/tez_join_hash.q.out
index 54ca9d2..1fd45aa 100644
--- a/ql/src/test/results/clientpositive/llap/tez_join_hash.q.out
+++ b/ql/src/test/results/clientpositive/llap/tez_join_hash.q.out
@@ -652,6 +652,10 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask
+ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask
+ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask
POSTHOOK: query: select key, count(*) from (select x.key as key, y.value as value from
srcpart x join srcpart y on (x.key = y.key)
union all
http://git-wip-us.apache.org/repos/asf/hive/blob/14bcbab1/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out b/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out
index 8d0aba1..2f51094 100644
--- a/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out
+++ b/ql/src/test/results/clientpositive/tez/tez_join_hash.q.out
@@ -638,6 +638,10 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask
+ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask
+ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask
POSTHOOK: query: select key, count(*) from (select x.key as key, y.value as value from
srcpart x join srcpart y on (x.key = y.key)
union all