You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/03/22 01:10:13 UTC
svn commit: r1580090 - in /pig/branches/branch-0.12: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
test/org/apache/pig/test/TestSecondarySort.java
Author: daijy
Date: Sat Mar 22 00:10:13 2014
New Revision: 1580090
URL: http://svn.apache.org/r1580090
Log:
PIG-3827: Custom partitioner is not picked up with secondary sort optimization
Modified:
pig/branches/branch-0.12/CHANGES.txt
pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
pig/branches/branch-0.12/test/org/apache/pig/test/TestSecondarySort.java
Modified: pig/branches/branch-0.12/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/CHANGES.txt?rev=1580090&r1=1580089&r2=1580090&view=diff
==============================================================================
--- pig/branches/branch-0.12/CHANGES.txt (original)
+++ pig/branches/branch-0.12/CHANGES.txt Sat Mar 22 00:10:13 2014
@@ -34,6 +34,8 @@ PIG-3480: TFile-based tmpfile compressio
BUG FIXES
+PIG-3827: Custom partitioner is not picked up with secondary sort optimization (daijy)
+
PIG-3826: Outer join with PushDownForEachFlatten generates wrong result (daijy)
PIG-3820: TestAvroStorage fail on some OS (daijy)
Modified: pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java?rev=1580090&r1=1580089&r2=1580090&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java (original)
+++ pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java Sat Mar 22 00:10:13 2014
@@ -152,6 +152,10 @@ public class SecondaryKeyOptimizer exten
if (mr.isGlobalSort())
return;
+ // Don't optimize when we already have a custom partitioner
+ if (mr.getCustomPartitioner()!=null)
+ return;
+
List<PhysicalOperator> mapLeaves = mr.mapPlan.getLeaves();
if (mapLeaves == null || mapLeaves.size() != 1) {
log
Modified: pig/branches/branch-0.12/test/org/apache/pig/test/TestSecondarySort.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/test/TestSecondarySort.java?rev=1580090&r1=1580089&r2=1580090&view=diff
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ pig/branches/branch-0.12/test/org/apache/pig/test/TestSecondarySort.java Sat Mar 22 00:10:13 2014
@@ -457,5 +457,36 @@ public class TestSecondarySort {
Util.deleteFile(cluster, clusterFilePath);
}
+
+ @Test
+ // Once custom partitioner is used, we cannot use secondary key optimizer, see PIG-3827
+ public void testCustomPartitionerWithSort() throws Exception {
+ File tmpFile1 = Util.createTempFileDelOnExit("test", "txt");
+ PrintStream ps1 = new PrintStream(new FileOutputStream(tmpFile1));
+ ps1.println("1\t2\t3");
+ ps1.println("1\t3\t4");
+ ps1.println("1\t4\t4");
+ ps1.println("1\t2\t4");
+ ps1.println("1\t8\t4");
+ ps1.println("2\t3\t4");
+ ps1.close();
+
+ String clusterPath = Util.removeColon(tmpFile1.getCanonicalPath());
+
+ Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), clusterPath);
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(clusterPath) + "' AS (a0, a1, a2);");
+ pigServer.registerQuery("B = group A by $0 PARTITION BY org.apache.pig.test.utils.WrongCustomPartitioner parallel 2;");
+ pigServer.registerQuery("C = foreach B { D = order A by a1 desc; generate group, D;};");
+ boolean captureException = false;
+ try {
+ pigServer.openIterator("C");
+ } catch (Exception e) {
+ captureException = true;
+ }
+
+ assertTrue(captureException);
+
+ Util.deleteFile(cluster, clusterPath);
+ }
}