svn commit: r1070945 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql: exec/ExecDriver.java history/HiveHistory.java

[nz...@apache.org]

Author: nzhang
Date: Tue Feb 15 15:46:13 2011
New Revision: 1070945

URL: http://svn.apache.org/viewvc?rev=1070945&view=rev
Log:
HIVE-1991 Hive Shell to output number of mappers and number of reducers (Siying Dong via Ning Zhang)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1070945&r1=1070944&r2=1070945&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Tue Feb 15 15:46:13 2011
@@ -86,6 +86,7 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Partitioner;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hadoop.mapred.TaskCompletionEvent;
+import org.apache.hadoop.mapred.TaskReport;
 import org.apache.log4j.Appender;
 import org.apache.log4j.BasicConfigurator;
 import org.apache.log4j.FileAppender;
@@ -333,6 +334,7 @@ public class ExecDriver extends Task<Map
     StringBuilder errMsg = new StringBuilder();
     long pullInterval = HiveConf.getLongVar(job, HiveConf.ConfVars.HIVECOUNTERSPULLINTERVAL);
     boolean initializing = true;
+    boolean initOutputPrinted = false;
     while (!rj.isComplete()) {
       try {
         Thread.sleep(pullInterval);
@@ -348,6 +350,41 @@ public class ExecDriver extends Task<Map
         initializing = false;
       }
 
+      if (!initOutputPrinted) {
+        SessionState ss = SessionState.get();
+
+        String logMapper;
+        String logReducer;
+
+        TaskReport[] mappers = jc.getMapTaskReports(rj.getJobID());
+        if (mappers == null) {
+          logMapper = "no information for number of mappers; ";
+        } else {
+          int numMap = mappers.length;
+          if (ss != null) {
+            ss.getHiveHistory().setTaskProperty(SessionState.get().getQueryId(), getId(),
+                Keys.TASK_NUM_MAPPERS, Integer.toString(numMap));
+          }
+          logMapper = "number of mappers: " + numMap + "; ";
+        }
+
+        TaskReport[] reducers = jc.getReduceTaskReports(rj.getJobID());
+        if (reducers == null) {
+          logReducer = "no information for number of reducers. ";
+        } else {
+          int numReduce = reducers.length;
+          if (ss != null) {
+            ss.getHiveHistory().setTaskProperty(SessionState.get().getQueryId(), getId(),
+                Keys.TASK_NUM_REDUCERS, Integer.toString(numReduce));
+          }
+          logReducer = "number of reducers: " + numReduce;
+        }
+
+        console
+            .printInfo("Hadoop job information for " + getId() + ": " + logMapper + logReducer);
+        initOutputPrinted = true;
+      }
+
       RunningJob newRj = jc.getJob(rj.getJobID());
       if (newRj == null) {
         // under exceptional load, hadoop may not be able to look up status

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java?rev=1070945&r1=1070944&r2=1070945&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java Tue Feb 15 15:46:13 2011
@@ -99,6 +99,7 @@ public class HiveHistory {
     TASK_HADOOP_ID,
     TASK_HADOOP_PROGRESS,
     TASK_COUNTERS,
+    TASK_NUM_MAPPERS,
     TASK_NUM_REDUCERS,
     ROWS_INSERTED
   };