You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/09/18 16:16:39 UTC

svn commit: r1625989 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/tez/ test/org/apache/pig/test/ test/org/apache/pig/test/data/GoldenFiles/ test/org/apache/pig/test/data/GoldenFiles/tez/ test/org/apache/pig/...

Author: rohini
Date: Thu Sep 18 14:16:39 2014
New Revision: 1625989

URL: http://svn.apache.org/r1625989
Log:
PIG-4069: Limit reduce task should start as soon as one map task finishes (rohini)

Added:
    pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-1.gld
      - copied unchanged from r1625962, pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld
    pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld
    pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld
Removed:
    pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/Main.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java
    pig/trunk/test/org/apache/pig/test/TestLimitVariable.java
    pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Sep 18 14:16:39 2014
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
  
 IMPROVEMENTS
 
+PIG-4069: Limit reduce task should start as soon as one map task finishes (rohini)
+
 PIG-4141: Ship UDF/LoadFunc/StoreFunc dependent jar automatically (daijy)
 
 PIG-4146: Create a target to run mr and tez unit test in one shot (daijy)

Modified: pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Thu Sep 18 14:16:39 2014
@@ -79,6 +79,11 @@ import org.apache.pig.tools.pigstats.Pig
 import org.apache.pig.tools.pigstats.PigStatsUtil;
 import org.apache.pig.tools.pigstats.ScriptState;
 import org.apache.pig.tools.timer.PerformanceTimerFactory;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.Period;
+import org.joda.time.PeriodType;
+import org.joda.time.format.PeriodFormat;
 
 /**
  * Main class for Pig engine.
@@ -161,7 +166,18 @@ public class Main {
      * @throws IOException
      */
     public static void main(String args[]) {
-        System.exit(run(args, null));
+        DateTime startTime = new DateTime();
+
+        int exitcode = run(args, null);
+
+        DateTime endTime = new DateTime();
+        Duration duration = new Duration(startTime, endTime);
+        Period period = duration.toPeriod().normalizedStandard(PeriodType.time());
+        log.info("Pig script completed in "
+                + PeriodFormat.getDefault().print(period)
+                + " (" + duration.getMillis() + " ms)");
+
+        System.exit(exitcode);
     }
 
     static int run(String args[], PigProgressNotificationListener listener) {

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java Thu Sep 18 14:16:39 2014
@@ -280,6 +280,9 @@ public class TezCompiler extends PhyPlan
                         from.plan.remove(from.plan.getOperator(store.getOperatorKey()));
                         from.plan.addAsLeaf(output);
                         storeSeen.put(store, output);
+
+                        //Remove unused store filename
+                        userFunc.getInputs().remove(1);
                     }
 
                     TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, from, tezOp);
@@ -341,7 +344,7 @@ public class TezCompiler extends PhyPlan
                     POValueOutputTez valueOutput = new POValueOutputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
                     oper.plan.addAsLeaf(valueOutput);
                     oper.setSplitter(true);
-    
+
                     // Create a splittee of store only
                     TezOperator storeOnlyTezOperator = getTezOp();
                     PhysicalPlan storeOnlyPhyPlan = new PhysicalPlan();
@@ -352,14 +355,14 @@ public class TezCompiler extends PhyPlan
                     storeOnlyTezOperator.plan = storeOnlyPhyPlan;
                     tezPlan.add(storeOnlyTezOperator);
                     phyToTezOpMap.put(store, storeOnlyTezOperator);
-    
+
                     // Create new operator as second splittee
                     curTezOp = getTezOp();
                     POValueInputTez valueInput2 = new POValueInputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
                     valueInput2.setInputKey(oper.getOperatorKey().toString());
                     curTezOp.plan.add(valueInput2);
                     tezPlan.add(curTezOp);
-    
+
                     // Connect splitter to splittee
                     TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, oper, storeOnlyTezOperator);
                     TezCompilerUtil.configureValueOnlyTupleOutput(edge,  DataMovementType.ONE_TO_ONE);
@@ -780,6 +783,8 @@ public class TezCompiler extends PhyPlan
                 phyToTezOpMap.put(op, curTezOp);
             }
 
+            boolean limitAfterSort = curTezOp.isGlobalSort();
+
             // If the parallelism of the current vertex is one and it doesn't do a LOAD (whose
             // parallelism is determined by the InputFormat), we don't need another vertex.
             if (curTezOp.getRequestedParallelism() == 1) {
@@ -791,6 +796,11 @@ public class TezCompiler extends PhyPlan
                     }
                 }
                 if (canStop) {
+                    if (limitAfterSort) {
+                        curTezOp.markLimitAfterSort();
+                    } else {
+                        curTezOp.markLimit();
+                    }
                     return;
                 }
             }
@@ -813,7 +823,8 @@ public class TezCompiler extends PhyPlan
             TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.SCATTER_GATHER);
 
             // Limit after order by with scalar expression
-            if (this.plan.getPredecessors(op).get(0) instanceof POSort) {
+            if (limitAfterSort) {
+                curTezOp.markLimitAfterSort();
                 output.setTaskIndexWithRecordIndexAsKey(true);
                 // POValueOutputTez will write key (task index, record index) in
                 // sorted order. So using UnorderedKVOutput instead of OrderedPartitionedKVOutput.
@@ -822,6 +833,8 @@ public class TezCompiler extends PhyPlan
                 edge.inputClassName = OrderedGroupedKVInput.class.getName();
                 edge.setIntermediateOutputKeyClass(TezCompilerUtil.TUPLE_CLASS);
                 edge.setIntermediateOutputKeyComparatorClass(PigTupleWritableComparator.class.getName());
+            } else {
+                curTezOp.markLimit();
             }
 
             // Then add a POValueInputTez to the start of the new tezOp.
@@ -1645,7 +1658,7 @@ public class TezCompiler extends PhyPlan
                 edge.partitionerClass = SkewedPartitionerTez.class;
             }
 
-            joinJobs[2].setSkewedJoin(true);
+            joinJobs[2].markSkewedJoin();
             sampleJobPair.first.sortOperator = joinJobs[2];
 
             if (rp == -1) {
@@ -2073,13 +2086,11 @@ public class TezCompiler extends PhyPlan
         oper1.markSampleBasedPartitioner();
 
         TezOperator oper2 = getTezOp();
-        oper2.setGlobalSort(true);
+        oper2.markGlobalSort();
         opers[1] = oper2;
         tezPlan.add(oper2);
 
         long limit = sort.getLimit();
-        //TODO: TezOperator limit not used at all
-        oper2.limit = limit;
 
         boolean[] sortOrder;
 
@@ -2261,6 +2272,7 @@ public class TezCompiler extends PhyPlan
 
                 // Explicitly set the parallelism for the new vertex to 1.
                 limitOper.setRequestedParallelism(1);
+                limitOper.markLimitAfterSort();
 
                 edge = TezCompilerUtil.connect(tezPlan, sortOpers[1], limitOper);
                 // LIMIT in this case should be ordered. So we output unordered with key as task index
@@ -2367,7 +2379,7 @@ public class TezCompiler extends PhyPlan
             // which unions input from the two predecessor vertices
             TezOperator unionTezOp = getTezOp();
             tezPlan.add(unionTezOp);
-            unionTezOp.setUnion();
+            unionTezOp.markUnion();
             unionTezOp.setRequestedParallelism(op.getRequestedParallelism());
             POShuffledValueInputTez unionInput =  new POShuffledValueInputTez(OperatorKey.genOpKey(scope));
             unionTezOp.plan.addAsLeaf(unionInput);

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java Thu Sep 18 14:16:39 2014
@@ -39,7 +39,6 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.pig.PigConfiguration;
 import org.apache.pig.PigException;
 import org.apache.pig.StoreFuncInterface;
 import org.apache.pig.backend.executionengine.ExecException;
@@ -632,14 +631,16 @@ public class TezDagBuilder extends TezOp
             new PigOutputFormat().checkOutputSpecs(job);
         }
 
+        String vmPluginName = null;
+        Configuration vmPluginConf = null;
+
         // Set the right VertexManagerPlugin
         if (tezOp.getEstimatedParallelism() != -1) {
             if (tezOp.isGlobalSort()||tezOp.isSkewedJoin()) {
                 // Set VertexManagerPlugin to PartitionerDefinedVertexManager, which is able
                 // to decrease/increase parallelism of sorting vertex dynamically
                 // based on the numQuantiles calculated by sample aggregation vertex
-                vertex.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
-                        PartitionerDefinedVertexManager.class.getName()));
+                vmPluginName = PartitionerDefinedVertexManager.class.getName();
                 log.info("Set VertexManagerPlugin to PartitionerDefinedParallelismVertexManager for vertex " + tezOp.getOperatorKey().toString());
             } else {
                 boolean containScatterGather = false;
@@ -655,9 +656,8 @@ public class TezDagBuilder extends TezOp
                 if (containScatterGather && !containCustomPartitioner) {
                     // Use auto-parallelism feature of ShuffleVertexManager to dynamically
                     // reduce the parallelism of the vertex
-                    VertexManagerPluginDescriptor vmPluginDescriptor = VertexManagerPluginDescriptor.create(
-                            ShuffleVertexManager.class.getName());
-                    Configuration vmPluginConf = ConfigurationUtil.toConfiguration(pc.getProperties(), false);
+                    vmPluginName = ShuffleVertexManager.class.getName();
+                    vmPluginConf = (vmPluginConf == null) ? ConfigurationUtil.toConfiguration(pc.getProperties(), false) : vmPluginConf;
                     vmPluginConf.setBoolean(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL, true);
                     if (vmPluginConf.getLong(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM,
                             InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER)!=
@@ -666,13 +666,37 @@ public class TezDagBuilder extends TezOp
                                 vmPluginConf.getLong(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM,
                                         InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER));
                     }
-                    vmPluginDescriptor.setUserPayload(TezUtils.createUserPayloadFromConf(vmPluginConf));
-                    vertex.setVertexManagerPlugin(vmPluginDescriptor);
                     log.info("Set auto parallelism for vertex " + tezOp.getOperatorKey().toString());
                 }
             }
         }
-
+/* TODO: Uncomment after TEZ-1590 is fixed
+        if (tezOp.isLimit() && (vmPluginName == null || vmPluginName.equals(ShuffleVertexManager.class.getName()))) {
+            if (tezOp.inEdges.values().iterator().next().inputClassName.equals(UnorderedKVInput.class.getName())) {
+                // Setting SRC_FRACTION to 0.00001 so that even if there are 100K source tasks,
+                // limit job starts when 1 source task finishes.
+                // If limit is part of a group by or join because their parallelism is 1,
+                // we should leave the configuration with the defaults.
+                vmPluginName = ShuffleVertexManager.class.getName();
+                vmPluginConf = (vmPluginConf == null) ? ConfigurationUtil.toConfiguration(pc.getProperties(), false) : vmPluginConf;
+                vmPluginConf.set(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, "0.00001");
+                vmPluginConf.set(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, "0.00001");
+                log.info("Set " + ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION + " to 0.00001 for limit vertex " + tezOp.getOperatorKey().toString());
+            }
+        }
+*/
+        // else if(tezOp.isLimitAfterSort())
+        // TODO: PIG-4049 If standalone Limit we need a new VertexManager or new input
+        // instead of ShuffledMergedInput. For limit part of the sort (order by parallel 1) itself
+        // need to enhance PartitionerDefinedVertexManager
+
+        if (vmPluginName != null) {
+            VertexManagerPluginDescriptor vmPluginDescriptor = VertexManagerPluginDescriptor.create(vmPluginName);
+            if (vmPluginConf != null) {
+                vmPluginDescriptor.setUserPayload(TezUtils.createUserPayloadFromConf(vmPluginConf));
+            }
+            vertex.setVertexManagerPlugin(vmPluginDescriptor);
+        }
         // Reset udfcontext jobconf. It is not supposed to be set in the front end
         UDFContext.getUDFContext().addJobConf(null);
         return vertex;

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java Thu Sep 18 14:16:39 2014
@@ -19,6 +19,7 @@ package org.apache.pig.backend.hadoop.ex
 
 import java.io.ByteArrayOutputStream;
 import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -65,7 +66,7 @@ public class TezOperator extends Operato
     // even when parallelism of source vertex changes.
     // Can change to int and set to -1 if TEZ-800 gets fixed.
     private AtomicInteger requestedParallelism = new AtomicInteger(-1);
-    
+
     private int estimatedParallelism = -1;
 
     // This is the parallelism of the vertex, it take account of:
@@ -94,25 +95,10 @@ public class TezOperator extends Operato
     // The result is two or more DAG connected DAG inside the same plan container
     boolean segmentBelow = false;
 
-    // Indicates if this is a limit after a sort
-    boolean limitAfterSort = false;
-
-    //Indicates if this job is an order by job
-    boolean globalSort = false;
-
-    //Indicate if this job is a union job 
-    boolean union = false;
-
     //The sort order of the columns;
     //asc is true and desc is false
     boolean[] sortOrder;
 
-    // Last POLimit value in this map reduce operator, needed by LimitAdjuster
-    // to add additional map reduce operator with 1 reducer after this
-    long limit = -1;
-
-    private boolean skewedJoin = false;
-
     // Flag to indicate if the small input splits need to be combined to form a larger
     // one in order to reduce the number of mappers. For merge join, both tables
     // are NOT combinable for correctness.
@@ -136,7 +122,6 @@ public class TezOperator extends Operato
 
     // Types of blocking operators. For now, we only support the following ones.
     private static enum OPER_FEATURE {
-        NONE,
         // Indicate if this job is a merge indexer
         INDEXER,
         // Indicate if this job is a sampling job
@@ -145,17 +130,30 @@ public class TezOperator extends Operato
         SAMPLE_AGGREGATOR,
         // Indicate if this job is a sample based partition job (order by/skewed join)
         SAMPLE_BASED_PARTITIONER,
+        // Indicate if this job is a global sort
+        GLOBAL_SORT,
         // Indicate if this job is a group by job
         GROUPBY,
         // Indicate if this job is a cogroup job
         COGROUP,
         // Indicate if this job is a regular join job
         HASHJOIN,
+        // Indicate if this job is a skewed join job
+        SKEWEDJOIN,
+        // Indicate if this job is a limit job
+        LIMIT,
+        // Indicate if this job is a limit job after sort
+        LIMIT_AFTER_SORT,
+        // Indicate if this job is a union job
+        UNION,
         // Indicate if this job is a native job
         NATIVE;
     };
 
-    OPER_FEATURE feature = OPER_FEATURE.NONE;
+    // Features in the job/vertex. Mostly will be only one feature.
+    // But in some cases can have more than one.
+    // For eg: a vertex can be both GLOBAL SORT and LIMIT if parallelism is 1
+    BitSet feature = new BitSet();
 
     private List<OperatorKey> vertexGroupMembers;
     // For union
@@ -279,76 +277,108 @@ public class TezOperator extends Operato
         this.closed = closed;
     }
 
+    public boolean isIndexer() {
+        return feature.get(OPER_FEATURE.INDEXER.ordinal());
+    }
+
+    public void markIndexer() {
+        feature.set(OPER_FEATURE.INDEXER.ordinal());
+    }
+
+    public boolean isSampler() {
+        return feature.get(OPER_FEATURE.SAMPLER.ordinal());
+    }
+
+    public void markSampler() {
+        feature.set(OPER_FEATURE.SAMPLER.ordinal());
+    }
+
+    public boolean isSampleAggregation() {
+        return feature.get(OPER_FEATURE.SAMPLE_AGGREGATOR.ordinal());
+    }
+
+    public void markSampleAggregation() {
+        feature.set(OPER_FEATURE.SAMPLE_AGGREGATOR.ordinal());
+    }
+
+    public boolean isSampleBasedPartitioner() {
+        return feature.get(OPER_FEATURE.SAMPLE_BASED_PARTITIONER.ordinal());
+    }
+
+    public void markSampleBasedPartitioner() {
+        feature.set(OPER_FEATURE.SAMPLE_BASED_PARTITIONER.ordinal());
+    }
+
+    public boolean isGlobalSort() {
+        return feature.get(OPER_FEATURE.GLOBAL_SORT.ordinal());
+    }
+
+    public void markGlobalSort() {
+        feature.set(OPER_FEATURE.GLOBAL_SORT.ordinal());
+    }
+
     public boolean isGroupBy() {
-        return (feature == OPER_FEATURE.GROUPBY);
+        return feature.get(OPER_FEATURE.GROUPBY.ordinal());
     }
 
     public void markGroupBy() {
-        feature = OPER_FEATURE.GROUPBY;
+        feature.set(OPER_FEATURE.GROUPBY.ordinal());
     }
 
     public boolean isCogroup() {
-        return (feature == OPER_FEATURE.COGROUP);
+        return feature.get(OPER_FEATURE.COGROUP.ordinal());
     }
 
     public void markCogroup() {
-        feature = OPER_FEATURE.COGROUP;
+        feature.set(OPER_FEATURE.COGROUP.ordinal());
     }
 
     public boolean isRegularJoin() {
-        return (feature == OPER_FEATURE.HASHJOIN);
+        return feature.get(OPER_FEATURE.HASHJOIN.ordinal());
     }
 
     public void markRegularJoin() {
-        feature = OPER_FEATURE.HASHJOIN;
+        feature.set(OPER_FEATURE.HASHJOIN.ordinal());
     }
 
-    public boolean isNative() {
-        return (feature == OPER_FEATURE.NATIVE);
+    public boolean isSkewedJoin() {
+        return feature.get(OPER_FEATURE.SKEWEDJOIN.ordinal());
     }
 
-    public void markNative() {
-        feature = OPER_FEATURE.NATIVE;
+    public void markSkewedJoin() {
+        feature.set(OPER_FEATURE.SKEWEDJOIN.ordinal());
     }
 
-    public boolean isUnion() {
-        return union;
+    public boolean isLimit() {
+        return feature.get(OPER_FEATURE.LIMIT.ordinal());
     }
 
-    public void setUnion() {
-        union = true;
+    public void markLimit() {
+        feature.set(OPER_FEATURE.LIMIT.ordinal());
     }
 
-    public boolean isIndexer() {
-        return (feature == OPER_FEATURE.INDEXER);
+    public boolean isLimitAfterSort() {
+        return feature.get(OPER_FEATURE.LIMIT_AFTER_SORT.ordinal());
     }
 
-    public void markIndexer() {
-        feature = OPER_FEATURE.INDEXER;
+    public void markLimitAfterSort() {
+        feature.set(OPER_FEATURE.LIMIT_AFTER_SORT.ordinal());
     }
 
-    public boolean isSampler() {
-        return (feature == OPER_FEATURE.SAMPLER);
-    }
-
-    public void markSampler() {
-        feature = OPER_FEATURE.SAMPLER;
+    public boolean isUnion() {
+        return feature.get(OPER_FEATURE.UNION.ordinal());
     }
 
-    public boolean isSampleAggregation() {
-        return (feature == OPER_FEATURE.SAMPLE_AGGREGATOR);
+    public void markUnion() {
+        feature.set(OPER_FEATURE.UNION.ordinal());
     }
 
-    public void markSampleAggregation() {
-        feature = OPER_FEATURE.SAMPLE_AGGREGATOR;
-    }
-    
-    public boolean isSampleBasedPartitioner() {
-        return (feature == OPER_FEATURE.SAMPLE_BASED_PARTITIONER);
+    public boolean isNative() {
+        return feature.get(OPER_FEATURE.NATIVE.ordinal());
     }
 
-    public void markSampleBasedPartitioner() {
-        feature = OPER_FEATURE.SAMPLE_BASED_PARTITIONER;
+    public void markNative() {
+        feature.set(OPER_FEATURE.NATIVE.ordinal());
     }
 
     public void setNeedEstimatedQuantile(boolean needEstimateParallelism) {
@@ -473,30 +503,6 @@ public class TezOperator extends Operato
         return sortOrder;
     }
 
-    public void setGlobalSort(boolean globalSort) {
-        this.globalSort = globalSort;
-    }
-
-    public boolean isGlobalSort() {
-        return globalSort;
-    }
-
-    public void setLimitAfterSort(boolean limitAfterSort) {
-        this.limitAfterSort = limitAfterSort;
-    }
-
-    public boolean isLimitAfterSort() {
-        return limitAfterSort;
-    }
-
-    public void setSkewedJoin(boolean skewedJoin) {
-        this.skewedJoin = skewedJoin;
-    }
-
-    public boolean isSkewedJoin() {
-        return skewedJoin;
-    }
-
     protected void noCombineSmallSplits() {
         combineSmallSplits = false;
     }

Modified: pig/trunk/test/org/apache/pig/test/TestLimitVariable.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLimitVariable.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLimitVariable.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLimitVariable.java Thu Sep 18 14:16:39 2014
@@ -20,16 +20,18 @@ package org.apache.pig.test;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigImplConstants;
 import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.util.ObjectSerializer;
 import org.apache.pig.newplan.logical.visitor.ScalarVariableValidator;
 import org.junit.AfterClass;
-import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -116,6 +118,34 @@ public class TestLimitVariable {
         Util.checkQueryOutputs(itE, expectedResE);
     }
 
+    @Test
+    public void testLimitVariable4() throws IOException {
+        String query =
+            "a = load '" + inputFile.getName() + "' as (x:int, y:int);" +
+            "b = group a all;" +
+            "c = foreach b generate COUNT(a) as sum;" +
+            "d = order a by $0 DESC;" +
+            "e = filter d by $0 != 4;" +
+            "f = limit e c.sum/2;" // return top half of the tuples
+            ;
+
+        try {
+            HashSet<String> disabledOptimizerRules = new HashSet<String>();
+            disabledOptimizerRules.add("PushUpFilter");
+            pigServer.getPigContext().getProperties().setProperty(PigImplConstants.PIG_OPTIMIZER_RULES_KEY,
+                    ObjectSerializer.serialize(disabledOptimizerRules));
+            Util.registerMultiLineQuery(pigServer, query);
+            Iterator<Tuple> it = pigServer.openIterator("f");
+
+            // Even if push up filter is disabled order should be retained
+            List<Tuple> expectedRes = Util.getTuplesFromConstantTupleStrings(new String[] {
+                    "(6,15)", "(5,10)", "(3,10)" });
+            Util.checkQueryOutputs(it, expectedRes);
+        } finally {
+            pigServer.getPigContext().getProperties().remove(PigImplConstants.PIG_OPTIMIZER_RULES_KEY);
+        }
+    }
+
     @Test(expected=FrontendException.class)
     public void testLimitVariableException1() throws Throwable {
         String query =

Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld?rev=1625989&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld Thu Sep 18 14:16:39 2014
@@ -0,0 +1,89 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-49
+#--------------------------------------------------
+Tez vertex scope-12	->	Tez vertex scope-33,Tez vertex scope-22,
+Tez vertex scope-22	->	Tez vertex scope-33,
+Tez vertex scope-33	->	Tez vertex scope-35,
+Tez vertex scope-35	->	Tez vertex scope-46,
+Tez vertex scope-46
+
+Tez vertex scope-12
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-15	->	 scope-22
+|   |
+|   Constant(DummyVal) - scope-14
+|
+|---New For Each(false,false,true)[tuple] - scope-21
+    |   |
+    |   Project[int][0] - scope-8
+    |   |
+    |   Project[int][1] - scope-9
+    |   |
+    |   POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-20
+    |   |
+    |   |---Project[tuple][*] - scope-19
+    |
+    |---ReservoirSample - scope-18
+        |
+        |---b: Local Rearrange[tuple]{tuple}(false) - scope-13	->	 scope-33
+            |   |
+            |   Project[int][0] - scope-8
+            |   |
+            |   Project[int][1] - scope-9
+            |
+            |---a: New For Each(false,false)[bag] - scope-7
+                |   |
+                |   Cast[int] - scope-2
+                |   |
+                |   |---Project[bytearray][0] - scope-1
+                |   |
+                |   Cast[int] - scope-5
+                |   |
+                |   |---Project[bytearray][1] - scope-4
+                |
+                |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-22
+# Plan on vertex
+POValueOutputTez - scope-32	->	 [scope-33]
+|
+|---New For Each(false)[tuple] - scope-31
+    |   |
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-30
+    |   |
+    |   |---Project[tuple][*] - scope-29
+    |
+    |---New For Each(false,false)[tuple] - scope-28
+        |   |
+        |   Constant(-1) - scope-27
+        |   |
+        |   Project[bag][1] - scope-24
+        |
+        |---Package(Packager)[tuple]{bytearray} - scope-23
+Tez vertex scope-33
+# Plan on vertex
+POIdentityInOutTez - scope-34	<-	 scope-12	->	 scope-35
+|   |
+|   Project[int][0] - scope-8
+|   |
+|   Project[int][1] - scope-9
+Tez vertex scope-35
+# Plan on vertex
+POValueOutputTez - scope-45	->	 [scope-46]
+|
+|---Limit - scope-44
+    |
+    |---New For Each(true)[tuple] - scope-43
+        |   |
+        |   Project[bag][1] - scope-42
+        |
+        |---Package(LitePackager)[tuple]{tuple} - scope-41
+Tez vertex scope-46
+# Plan on vertex
+c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-11
+|
+|---Limit - scope-48
+    |
+    |---POValueInputTez - scope-47	<-	 scope-35

Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld?rev=1625989&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld Thu Sep 18 14:16:39 2014
@@ -0,0 +1,151 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-71
+#--------------------------------------------------
+Tez vertex scope-33	->	Tez vertex scope-37,Tez vertex scope-60,Tez vertex scope-49,
+Tez vertex scope-49	->	Tez vertex scope-60,
+Tez vertex scope-60	->	Tez vertex scope-62,
+Tez vertex scope-37	->	Tez vertex scope-67,Tez vertex scope-62,
+Tez vertex scope-62	->	Tez vertex scope-67,
+Tez vertex scope-67
+
+Tez vertex scope-33
+# Plan on vertex
+a: Split - scope-92
+|   |
+|   g: Local Rearrange[tuple]{chararray}(false) - scope-83	->	 scope-37
+|   |   |
+|   |   Project[chararray][0] - scope-85
+|   |
+|   |---h: New For Each(false,false)[bag] - scope-72
+|       |   |
+|       |   Project[chararray][0] - scope-73
+|       |   |
+|       |   POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-74
+|       |   |
+|       |   |---Project[bag][1] - scope-75
+|       |
+|       |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-86
+|   |
+|   Local Rearrange[tuple]{tuple}(false) - scope-42	->	 scope-49
+|   |   |
+|   |   Constant(DummyVal) - scope-41
+|   |
+|   |---New For Each(false,false,true)[tuple] - scope-48
+|       |   |
+|       |   Project[int][0] - scope-11
+|       |   |
+|       |   Project[int][1] - scope-12
+|       |   |
+|       |   POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-47
+|       |   |
+|       |   |---Project[tuple][*] - scope-46
+|       |
+|       |---ReservoirSample - scope-45
+|           |
+|           |---b: Local Rearrange[tuple]{tuple}(false) - scope-40	->	 scope-60
+|               |   |
+|               |   Project[int][0] - scope-11
+|               |   |
+|               |   Project[int][1] - scope-12
+|
+|---a: New For Each(false,false)[bag] - scope-7
+    |   |
+    |   Cast[int] - scope-2
+    |   |
+    |   |---Project[bytearray][0] - scope-1
+    |   |
+    |   Cast[int] - scope-5
+    |   |
+    |   |---Project[bytearray][1] - scope-4
+    |
+    |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-49
+# Plan on vertex
+POValueOutputTez - scope-59	->	 [scope-60]
+|
+|---New For Each(false)[tuple] - scope-58
+    |   |
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-57
+    |   |
+    |   |---Project[tuple][*] - scope-56
+    |
+    |---New For Each(false,false)[tuple] - scope-55
+        |   |
+        |   Constant(-1) - scope-54
+        |   |
+        |   Project[bag][1] - scope-51
+        |
+        |---Package(Packager)[tuple]{bytearray} - scope-50
+Tez vertex scope-60
+# Plan on vertex
+POIdentityInOutTez - scope-61	<-	 scope-33	->	 scope-62
+|   |
+|   Project[int][0] - scope-11
+|   |
+|   Project[int][1] - scope-12
+Tez vertex scope-37
+# Combine plan on edge <scope-33>
+g: Local Rearrange[tuple]{chararray}(false) - scope-87	->	 scope-37
+|   |
+|   Project[chararray][0] - scope-89
+|
+|---h: New For Each(false,false)[bag] - scope-76
+    |   |
+    |   Project[chararray][0] - scope-77
+    |   |
+    |   POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-78
+    |   |
+    |   |---Project[bag][1] - scope-79
+    |
+    |---g: Package(CombinerPackager)[tuple]{chararray} - scope-82
+# Plan on vertex
+POValueOutputTez - scope-70	->	 [scope-67, scope-62]
+|
+|---h: New For Each(false)[bag] - scope-23
+    |   |
+    |   POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-21
+    |   |
+    |   |---Project[bag][1] - scope-80
+    |
+    |---g: Package(CombinerPackager)[tuple]{chararray} - scope-17
+Tez vertex scope-62
+# Plan on vertex
+POValueOutputTez - scope-66	->	 [scope-67]
+|
+|---c: Limit - scope-25
+    |   |
+    |   Divide[long] - scope-31
+    |   |
+    |   |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.ReadScalarsTez)[long] - scope-28
+    |   |   |
+    |   |   |---Constant(0) - scope-26
+    |   |
+    |   |---Cast[long] - scope-30
+    |       |
+    |       |---Constant(2) - scope-29
+    |
+    |---New For Each(true)[tuple] - scope-65
+        |   |
+        |   Project[bag][1] - scope-64
+        |
+        |---Package(LitePackager)[tuple]{tuple} - scope-63
+Tez vertex scope-67
+# Plan on vertex
+c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-32
+|
+|---c: Limit - scope-69
+    |   |
+    |   Divide[long] - scope-31
+    |   |
+    |   |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.ReadScalarsTez)[long] - scope-28
+    |   |   |
+    |   |   |---Constant(0) - scope-26
+    |   |
+    |   |---Cast[long] - scope-30
+    |       |
+    |       |---Constant(2) - scope-29
+    |
+    |---POValueInputTez - scope-68	<-	 scope-62

Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Thu Sep 18 14:16:39 2014
@@ -142,7 +142,31 @@ public class TestTezCompiler {
                 "c = foreach b generate y;" +
                 "store c into 'file:///tmp/output';";
 
-        run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld");
+        run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-1.gld");
+    }
+
+    @Test
+    public void testLimitOrderby() throws Exception {
+        String query =
+                "a = load 'file:///tmp/input' as (x:int, y:int);" +
+                "b = order a by x, y;" +
+                "c = limit b 10;" +
+                "store c into 'file:///tmp/output';";
+
+        run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld");
+    }
+
+    @Test
+    public void testLimitScalarOrderby() throws Exception {
+        String query =
+                "a = load 'file:///tmp/input' as (x:int, y:int);" +
+                "b = order a by x, y;" +
+                "g = group a all;" +
+                "h = foreach g generate COUNT(a) as sum;" +
+                "c = limit b h.sum/2;" +
+                "store c into 'file:///tmp/output';";
+
+        run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld");
     }
 
     @Test