svn commit: r1639392 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java test/org/apache/pig/test/TestFetch.java

[ch...@apache.org]

Author: cheolsoo
Date: Thu Nov 13 17:00:17 2014
New Revision: 1639392

URL: http://svn.apache.org/r1639392
Log:
PIG-4329: Fetch optimization should be disabled when limit is not pushed up (lbendig via cheolsoo)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java
    pig/trunk/test/org/apache/pig/test/TestFetch.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1639392&r1=1639391&r2=1639392&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Nov 13 17:00:17 2014
@@ -26,6 +26,8 @@ IMPROVEMENTS
  
 BUG FIXES
 
+PIG-4329: Fetch optimization should be disabled when limit is not pushed up (lbendig via cheolsoo)
+
 PIG-3413: JsonLoader fails the pig job in case of malformed json input (eyal via daijy)
 
 PIG-4247: S3 properties are not picked up from core-site.xml in local mode (cheolsoo)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java?rev=1639392&r1=1639391&r2=1639392&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java Thu Nov 13 17:00:17 2014
@@ -38,7 +38,6 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFRJoin;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POGlobalRearrange;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeCogroup;
@@ -98,16 +97,20 @@ public class FetchOptimizer {
             FetchablePlanVisitor fpv = new FetchablePlanVisitor(pc, pp);
             fpv.visit();
             // Plan is fetchable only if FetchablePlanVisitor returns true AND
-            // limit is present in the plan. Limit is a safeguard. If the input
-            // is large, and there is no limit, fetch optimizer will fetch the
-            // entire input to the client. That can be dangerous.
-            boolean isFetchable = fpv.isPlanFetchable() && 
-                    PlanHelper.containsPhysicalOperator(pp, POLimit.class);
-            if (isFetchable) {
-                pc.getProperties().setProperty(PigImplConstants.CONVERTED_TO_FETCH, "true");
-                init(pp);
+            // limit is present in the plan, i.e: limit is pushed up to the loader.
+            // Limit is a safeguard. If the input is large, and there is no limit, 
+            // fetch optimizer will fetch the entire input to the client. That can be dangerous.
+            if (!fpv.isPlanFetchable()) {
+                return false;
+            }
+            for (POLoad load : PlanHelper.getPhysicalOperators(pp, POLoad.class)) {
+                if (load.getLimit() == -1) {
+                    return false;
+                }
             }
-            return isFetchable;
+            pc.getProperties().setProperty(PigImplConstants.CONVERTED_TO_FETCH, "true");
+            init(pp);
+            return true;
         }
         return false;
     }

Modified: pig/trunk/test/org/apache/pig/test/TestFetch.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestFetch.java?rev=1639392&r1=1639391&r2=1639392&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestFetch.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestFetch.java Thu Nov 13 17:00:17 2014
@@ -127,7 +127,8 @@ public class TestFetch {
                 .compile(lp, null);
 
         boolean planFetchable = FetchOptimizer.isPlanFetchable(pigServer.getPigContext(), pp);
-        assertTrue(planFetchable);
+        //plan is not fetchable since limit is not pushed up to the loader
+        assertFalse(planFetchable);
 
     }