You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ya...@apache.org on 2010/12/09 19:38:51 UTC
svn commit: r1044075 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
Author: yanz
Date: Thu Dec 9 18:38:50 2010
New Revision: 1044075
URL: http://svn.apache.org/viewvc?rev=1044075&view=rev
Log:
PIG-1757: After split combination, the number of maps may vary slightly (yanz)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1044075&r1=1044074&r2=1044075&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Dec 9 18:38:50 2010
@@ -230,6 +230,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1757: After split combination, the number of maps may vary slightly (yanz)
+
PIG-1760: Need to report progress in all databags (rding)
PIG-1709: Skewed join use fewer reducer for extreme large key (daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java?rev=1044075&r1=1044074&r2=1044075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java Thu Dec 9 18:38:50 2010
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Comparator;
import java.util.Collections;
+import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -375,6 +376,8 @@ public class MapRedUtil {
} else {
ComparableSplit csplit = new ComparableSplit(split, comparableSplitId++);
String[] locations = split.getLocations();
+ // sort the locations to stabilize the number of maps: PIG-1757
+ Arrays.sort(locations);
HashSet<String> locationSeen = new HashSet<String>();
for (String location : locations)
{