You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2013/02/13 20:31:21 UTC
svn commit: r1445877 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java
Author: rohini
Date: Wed Feb 13 19:31:21 2013
New Revision: 1445877
URL: http://svn.apache.org/r1445877
Log:
PIG-3179: Task Information Header only prints out the first split for each task (knoguchi via rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1445877&r1=1445876&r2=1445877&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Feb 13 19:31:21 2013
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-3179: Task Information Header only prints out the first split for each task (knoguchi via rohini)
+
PIG-3108: HBaseStorage returns empty maps when mixing wildcard with other columns (christoph.bauer via billgraham)
PIG-3178: Print a stacktrace when ExecutableManager hits an OOM (knoguchi via rohini)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java?rev=1445877&r1=1445876&r2=1445877&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/streaming/HadoopExecutableManager.java Wed Feb 13 19:31:21 2013
@@ -199,15 +199,23 @@ public class HadoopExecutableManager ext
if (job.getBoolean("mapred.task.is.map", false)) {
MapContext context = (MapContext)PigMapReduce.sJobContext;
PigSplit pigSplit = (PigSplit)context.getInputSplit();
- InputSplit wrappedSplit = pigSplit.getWrappedSplit();
- if (wrappedSplit instanceof FileSplit) {
- FileSplit mapInputFileSplit = (FileSplit)wrappedSplit;
- processError("\nInput-split file: " +
- mapInputFileSplit.getPath().toString());
- processError("\nInput-split start-offset: " +
- Long.toString(mapInputFileSplit.getStart()));
- processError("\nInput-split length: " +
- Long.toString(mapInputFileSplit.getLength()));
+ int numPaths = pigSplit.getNumPaths();
+ processError("\nPigSplit contains " + numPaths + " wrappedSplits.");
+
+ StringBuilder sb = new StringBuilder();
+ for(int i = 0; i < numPaths; i++) {
+ InputSplit wrappedSplit = pigSplit.getWrappedSplit(i);
+ if (wrappedSplit instanceof FileSplit) {
+ FileSplit mapInputFileSplit = (FileSplit)wrappedSplit;
+ sb.append("\nInput-split: file=");
+ sb.append(mapInputFileSplit.getPath());
+ sb.append(" start-offset=");
+ sb.append(Long.toString(mapInputFileSplit.getStart()));
+ sb.append(" length=");
+ sb.append(Long.toString(mapInputFileSplit.getLength()));
+ processError(sb.toString());
+ sb.setLength(0);
+ }
}
}
processError("\n===== * * * =====\n");