You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2013/02/13 20:31:21 UTC

svn commit: r1445877 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java

Author: rohini
Date: Wed Feb 13 19:31:21 2013
New Revision: 1445877

URL: http://svn.apache.org/r1445877
Log:
PIG-3179: Task Information Header only prints out the first split for each task (knoguchi via rohini)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1445877&r1=1445876&r2=1445877&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Feb 13 19:31:21 2013
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-3179: Task Information Header only prints out the first split for each task (knoguchi via rohini)
+
 PIG-3108: HBaseStorage returns empty maps when mixing wildcard with other columns (christoph.bauer via billgraham)
 
 PIG-3178: Print a stacktrace when ExecutableManager hits an OOM (knoguchi via rohini)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java?rev=1445877&r1=1445876&r2=1445877&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java Wed Feb 13 19:31:21 2013
@@ -199,15 +199,23 @@ public class HadoopExecutableManager ext
         if (job.getBoolean("mapred.task.is.map", false)) {
             MapContext context = (MapContext)PigMapReduce.sJobContext;
             PigSplit pigSplit = (PigSplit)context.getInputSplit();
-            InputSplit wrappedSplit = pigSplit.getWrappedSplit();
-            if (wrappedSplit instanceof FileSplit) {
-                FileSplit mapInputFileSplit = (FileSplit)wrappedSplit;
-                processError("\nInput-split file: " + 
-                             mapInputFileSplit.getPath().toString());
-                processError("\nInput-split start-offset: " + 
-                             Long.toString(mapInputFileSplit.getStart()));
-                processError("\nInput-split length: " + 
-                             Long.toString(mapInputFileSplit.getLength()));
+            int numPaths = pigSplit.getNumPaths();
+            processError("\nPigSplit contains " + numPaths + " wrappedSplits.");
+
+            StringBuilder sb = new StringBuilder();
+            for(int i = 0; i < numPaths; i++) {
+              InputSplit wrappedSplit = pigSplit.getWrappedSplit(i);
+              if (wrappedSplit instanceof FileSplit) {
+                  FileSplit mapInputFileSplit = (FileSplit)wrappedSplit;
+                  sb.append("\nInput-split: file=");
+                  sb.append(mapInputFileSplit.getPath());
+                  sb.append(" start-offset=");
+                  sb.append(Long.toString(mapInputFileSplit.getStart()));
+                  sb.append(" length=");
+                  sb.append(Long.toString(mapInputFileSplit.getLength()));
+                  processError(sb.toString());
+                  sb.setLength(0);
+              }
             }
         }
         processError("\n=====          * * *          =====\n");