You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/09/18 16:16:39 UTC
svn commit: r1625989 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/tez/
test/org/apache/pig/test/ test/org/apache/pig/test/data/GoldenFiles/
test/org/apache/pig/test/data/GoldenFiles/tez/ test/org/apache/pig/...
Author: rohini
Date: Thu Sep 18 14:16:39 2014
New Revision: 1625989
URL: http://svn.apache.org/r1625989
Log:
PIG-4069: Limit reduce task should start as soon as one map task finishes (rohini)
Added:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-1.gld
- copied unchanged from r1625962, pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld
Removed:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/Main.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java
pig/trunk/test/org/apache/pig/test/TestLimitVariable.java
pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Sep 18 14:16:39 2014
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-4069: Limit reduce task should start as soon as one map task finishes (rohini)
+
PIG-4141: Ship UDF/LoadFunc/StoreFunc dependent jar automatically (daijy)
PIG-4146: Create a target to run mr and tez unit test in one shot (daijy)
Modified: pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Thu Sep 18 14:16:39 2014
@@ -79,6 +79,11 @@ import org.apache.pig.tools.pigstats.Pig
import org.apache.pig.tools.pigstats.PigStatsUtil;
import org.apache.pig.tools.pigstats.ScriptState;
import org.apache.pig.tools.timer.PerformanceTimerFactory;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.Period;
+import org.joda.time.PeriodType;
+import org.joda.time.format.PeriodFormat;
/**
* Main class for Pig engine.
@@ -161,7 +166,18 @@ public class Main {
* @throws IOException
*/
public static void main(String args[]) {
- System.exit(run(args, null));
+ DateTime startTime = new DateTime();
+
+ int exitcode = run(args, null);
+
+ DateTime endTime = new DateTime();
+ Duration duration = new Duration(startTime, endTime);
+ Period period = duration.toPeriod().normalizedStandard(PeriodType.time());
+ log.info("Pig script completed in "
+ + PeriodFormat.getDefault().print(period)
+ + " (" + duration.getMillis() + " ms)");
+
+ System.exit(exitcode);
}
static int run(String args[], PigProgressNotificationListener listener) {
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java Thu Sep 18 14:16:39 2014
@@ -280,6 +280,9 @@ public class TezCompiler extends PhyPlan
from.plan.remove(from.plan.getOperator(store.getOperatorKey()));
from.plan.addAsLeaf(output);
storeSeen.put(store, output);
+
+ //Remove unused store filename
+ userFunc.getInputs().remove(1);
}
TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, from, tezOp);
@@ -341,7 +344,7 @@ public class TezCompiler extends PhyPlan
POValueOutputTez valueOutput = new POValueOutputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
oper.plan.addAsLeaf(valueOutput);
oper.setSplitter(true);
-
+
// Create a splittee of store only
TezOperator storeOnlyTezOperator = getTezOp();
PhysicalPlan storeOnlyPhyPlan = new PhysicalPlan();
@@ -352,14 +355,14 @@ public class TezCompiler extends PhyPlan
storeOnlyTezOperator.plan = storeOnlyPhyPlan;
tezPlan.add(storeOnlyTezOperator);
phyToTezOpMap.put(store, storeOnlyTezOperator);
-
+
// Create new operator as second splittee
curTezOp = getTezOp();
POValueInputTez valueInput2 = new POValueInputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
valueInput2.setInputKey(oper.getOperatorKey().toString());
curTezOp.plan.add(valueInput2);
tezPlan.add(curTezOp);
-
+
// Connect splitter to splittee
TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, oper, storeOnlyTezOperator);
TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.ONE_TO_ONE);
@@ -780,6 +783,8 @@ public class TezCompiler extends PhyPlan
phyToTezOpMap.put(op, curTezOp);
}
+ boolean limitAfterSort = curTezOp.isGlobalSort();
+
// If the parallelism of the current vertex is one and it doesn't do a LOAD (whose
// parallelism is determined by the InputFormat), we don't need another vertex.
if (curTezOp.getRequestedParallelism() == 1) {
@@ -791,6 +796,11 @@ public class TezCompiler extends PhyPlan
}
}
if (canStop) {
+ if (limitAfterSort) {
+ curTezOp.markLimitAfterSort();
+ } else {
+ curTezOp.markLimit();
+ }
return;
}
}
@@ -813,7 +823,8 @@ public class TezCompiler extends PhyPlan
TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.SCATTER_GATHER);
// Limit after order by with scalar expression
- if (this.plan.getPredecessors(op).get(0) instanceof POSort) {
+ if (limitAfterSort) {
+ curTezOp.markLimitAfterSort();
output.setTaskIndexWithRecordIndexAsKey(true);
// POValueOutputTez will write key (task index, record index) in
// sorted order. So using UnorderedKVOutput instead of OrderedPartitionedKVOutput.
@@ -822,6 +833,8 @@ public class TezCompiler extends PhyPlan
edge.inputClassName = OrderedGroupedKVInput.class.getName();
edge.setIntermediateOutputKeyClass(TezCompilerUtil.TUPLE_CLASS);
edge.setIntermediateOutputKeyComparatorClass(PigTupleWritableComparator.class.getName());
+ } else {
+ curTezOp.markLimit();
}
// Then add a POValueInputTez to the start of the new tezOp.
@@ -1645,7 +1658,7 @@ public class TezCompiler extends PhyPlan
edge.partitionerClass = SkewedPartitionerTez.class;
}
- joinJobs[2].setSkewedJoin(true);
+ joinJobs[2].markSkewedJoin();
sampleJobPair.first.sortOperator = joinJobs[2];
if (rp == -1) {
@@ -2073,13 +2086,11 @@ public class TezCompiler extends PhyPlan
oper1.markSampleBasedPartitioner();
TezOperator oper2 = getTezOp();
- oper2.setGlobalSort(true);
+ oper2.markGlobalSort();
opers[1] = oper2;
tezPlan.add(oper2);
long limit = sort.getLimit();
- //TODO: TezOperator limit not used at all
- oper2.limit = limit;
boolean[] sortOrder;
@@ -2261,6 +2272,7 @@ public class TezCompiler extends PhyPlan
// Explicitly set the parallelism for the new vertex to 1.
limitOper.setRequestedParallelism(1);
+ limitOper.markLimitAfterSort();
edge = TezCompilerUtil.connect(tezPlan, sortOpers[1], limitOper);
// LIMIT in this case should be ordered. So we output unordered with key as task index
@@ -2367,7 +2379,7 @@ public class TezCompiler extends PhyPlan
// which unions input from the two predecessor vertices
TezOperator unionTezOp = getTezOp();
tezPlan.add(unionTezOp);
- unionTezOp.setUnion();
+ unionTezOp.markUnion();
unionTezOp.setRequestedParallelism(op.getRequestedParallelism());
POShuffledValueInputTez unionInput = new POShuffledValueInputTez(OperatorKey.genOpKey(scope));
unionTezOp.plan.addAsLeaf(unionInput);
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java Thu Sep 18 14:16:39 2014
@@ -39,7 +39,6 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.pig.PigConfiguration;
import org.apache.pig.PigException;
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.backend.executionengine.ExecException;
@@ -632,14 +631,16 @@ public class TezDagBuilder extends TezOp
new PigOutputFormat().checkOutputSpecs(job);
}
+ String vmPluginName = null;
+ Configuration vmPluginConf = null;
+
// Set the right VertexManagerPlugin
if (tezOp.getEstimatedParallelism() != -1) {
if (tezOp.isGlobalSort()||tezOp.isSkewedJoin()) {
// Set VertexManagerPlugin to PartitionerDefinedVertexManager, which is able
// to decrease/increase parallelism of sorting vertex dynamically
// based on the numQuantiles calculated by sample aggregation vertex
- vertex.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
- PartitionerDefinedVertexManager.class.getName()));
+ vmPluginName = PartitionerDefinedVertexManager.class.getName();
log.info("Set VertexManagerPlugin to PartitionerDefinedParallelismVertexManager for vertex " + tezOp.getOperatorKey().toString());
} else {
boolean containScatterGather = false;
@@ -655,9 +656,8 @@ public class TezDagBuilder extends TezOp
if (containScatterGather && !containCustomPartitioner) {
// Use auto-parallelism feature of ShuffleVertexManager to dynamically
// reduce the parallelism of the vertex
- VertexManagerPluginDescriptor vmPluginDescriptor = VertexManagerPluginDescriptor.create(
- ShuffleVertexManager.class.getName());
- Configuration vmPluginConf = ConfigurationUtil.toConfiguration(pc.getProperties(), false);
+ vmPluginName = ShuffleVertexManager.class.getName();
+ vmPluginConf = (vmPluginConf == null) ? ConfigurationUtil.toConfiguration(pc.getProperties(), false) : vmPluginConf;
vmPluginConf.setBoolean(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL, true);
if (vmPluginConf.getLong(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM,
InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER)!=
@@ -666,13 +666,37 @@ public class TezDagBuilder extends TezOp
vmPluginConf.getLong(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM,
InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER));
}
- vmPluginDescriptor.setUserPayload(TezUtils.createUserPayloadFromConf(vmPluginConf));
- vertex.setVertexManagerPlugin(vmPluginDescriptor);
log.info("Set auto parallelism for vertex " + tezOp.getOperatorKey().toString());
}
}
}
-
+/* TODO: Uncomment after TEZ-1590 is fixed
+ if (tezOp.isLimit() && (vmPluginName == null || vmPluginName.equals(ShuffleVertexManager.class.getName()))) {
+ if (tezOp.inEdges.values().iterator().next().inputClassName.equals(UnorderedKVInput.class.getName())) {
+ // Setting SRC_FRACTION to 0.00001 so that even if there are 100K source tasks,
+ // limit job starts when 1 source task finishes.
+ // If limit is part of a group by or join because their parallelism is 1,
+ // we should leave the configuration with the defaults.
+ vmPluginName = ShuffleVertexManager.class.getName();
+ vmPluginConf = (vmPluginConf == null) ? ConfigurationUtil.toConfiguration(pc.getProperties(), false) : vmPluginConf;
+ vmPluginConf.set(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, "0.00001");
+ vmPluginConf.set(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, "0.00001");
+ log.info("Set " + ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION + " to 0.00001 for limit vertex " + tezOp.getOperatorKey().toString());
+ }
+ }
+*/
+ // else if(tezOp.isLimitAfterSort())
+ // TODO: PIG-4049 If standalone Limit we need a new VertexManager or new input
+ // instead of ShuffledMergedInput. For limit part of the sort (order by parallel 1) itself
+ // need to enhance PartitionerDefinedVertexManager
+
+ if (vmPluginName != null) {
+ VertexManagerPluginDescriptor vmPluginDescriptor = VertexManagerPluginDescriptor.create(vmPluginName);
+ if (vmPluginConf != null) {
+ vmPluginDescriptor.setUserPayload(TezUtils.createUserPayloadFromConf(vmPluginConf));
+ }
+ vertex.setVertexManagerPlugin(vmPluginDescriptor);
+ }
// Reset udfcontext jobconf. It is not supposed to be set in the front end
UDFContext.getUDFContext().addJobConf(null);
return vertex;
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java Thu Sep 18 14:16:39 2014
@@ -19,6 +19,7 @@ package org.apache.pig.backend.hadoop.ex
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -65,7 +66,7 @@ public class TezOperator extends Operato
// even when parallelism of source vertex changes.
// Can change to int and set to -1 if TEZ-800 gets fixed.
private AtomicInteger requestedParallelism = new AtomicInteger(-1);
-
+
private int estimatedParallelism = -1;
// This is the parallelism of the vertex, it take account of:
@@ -94,25 +95,10 @@ public class TezOperator extends Operato
// The result is two or more DAG connected DAG inside the same plan container
boolean segmentBelow = false;
- // Indicates if this is a limit after a sort
- boolean limitAfterSort = false;
-
- //Indicates if this job is an order by job
- boolean globalSort = false;
-
- //Indicate if this job is a union job
- boolean union = false;
-
//The sort order of the columns;
//asc is true and desc is false
boolean[] sortOrder;
- // Last POLimit value in this map reduce operator, needed by LimitAdjuster
- // to add additional map reduce operator with 1 reducer after this
- long limit = -1;
-
- private boolean skewedJoin = false;
-
// Flag to indicate if the small input splits need to be combined to form a larger
// one in order to reduce the number of mappers. For merge join, both tables
// are NOT combinable for correctness.
@@ -136,7 +122,6 @@ public class TezOperator extends Operato
// Types of blocking operators. For now, we only support the following ones.
private static enum OPER_FEATURE {
- NONE,
// Indicate if this job is a merge indexer
INDEXER,
// Indicate if this job is a sampling job
@@ -145,17 +130,30 @@ public class TezOperator extends Operato
SAMPLE_AGGREGATOR,
// Indicate if this job is a sample based partition job (order by/skewed join)
SAMPLE_BASED_PARTITIONER,
+ // Indicate if this job is a global sort
+ GLOBAL_SORT,
// Indicate if this job is a group by job
GROUPBY,
// Indicate if this job is a cogroup job
COGROUP,
// Indicate if this job is a regular join job
HASHJOIN,
+ // Indicate if this job is a skewed join job
+ SKEWEDJOIN,
+ // Indicate if this job is a limit job
+ LIMIT,
+ // Indicate if this job is a limit job after sort
+ LIMIT_AFTER_SORT,
+ // Indicate if this job is a union job
+ UNION,
// Indicate if this job is a native job
NATIVE;
};
- OPER_FEATURE feature = OPER_FEATURE.NONE;
+ // Features in the job/vertex. Mostly will be only one feature.
+ // But in some cases can have more than one.
+ // For eg: a vertex can be both GLOBAL SORT and LIMIT if parallelism is 1
+ BitSet feature = new BitSet();
private List<OperatorKey> vertexGroupMembers;
// For union
@@ -279,76 +277,108 @@ public class TezOperator extends Operato
this.closed = closed;
}
+ public boolean isIndexer() {
+ return feature.get(OPER_FEATURE.INDEXER.ordinal());
+ }
+
+ public void markIndexer() {
+ feature.set(OPER_FEATURE.INDEXER.ordinal());
+ }
+
+ public boolean isSampler() {
+ return feature.get(OPER_FEATURE.SAMPLER.ordinal());
+ }
+
+ public void markSampler() {
+ feature.set(OPER_FEATURE.SAMPLER.ordinal());
+ }
+
+ public boolean isSampleAggregation() {
+ return feature.get(OPER_FEATURE.SAMPLE_AGGREGATOR.ordinal());
+ }
+
+ public void markSampleAggregation() {
+ feature.set(OPER_FEATURE.SAMPLE_AGGREGATOR.ordinal());
+ }
+
+ public boolean isSampleBasedPartitioner() {
+ return feature.get(OPER_FEATURE.SAMPLE_BASED_PARTITIONER.ordinal());
+ }
+
+ public void markSampleBasedPartitioner() {
+ feature.set(OPER_FEATURE.SAMPLE_BASED_PARTITIONER.ordinal());
+ }
+
+ public boolean isGlobalSort() {
+ return feature.get(OPER_FEATURE.GLOBAL_SORT.ordinal());
+ }
+
+ public void markGlobalSort() {
+ feature.set(OPER_FEATURE.GLOBAL_SORT.ordinal());
+ }
+
public boolean isGroupBy() {
- return (feature == OPER_FEATURE.GROUPBY);
+ return feature.get(OPER_FEATURE.GROUPBY.ordinal());
}
public void markGroupBy() {
- feature = OPER_FEATURE.GROUPBY;
+ feature.set(OPER_FEATURE.GROUPBY.ordinal());
}
public boolean isCogroup() {
- return (feature == OPER_FEATURE.COGROUP);
+ return feature.get(OPER_FEATURE.COGROUP.ordinal());
}
public void markCogroup() {
- feature = OPER_FEATURE.COGROUP;
+ feature.set(OPER_FEATURE.COGROUP.ordinal());
}
public boolean isRegularJoin() {
- return (feature == OPER_FEATURE.HASHJOIN);
+ return feature.get(OPER_FEATURE.HASHJOIN.ordinal());
}
public void markRegularJoin() {
- feature = OPER_FEATURE.HASHJOIN;
+ feature.set(OPER_FEATURE.HASHJOIN.ordinal());
}
- public boolean isNative() {
- return (feature == OPER_FEATURE.NATIVE);
+ public boolean isSkewedJoin() {
+ return feature.get(OPER_FEATURE.SKEWEDJOIN.ordinal());
}
- public void markNative() {
- feature = OPER_FEATURE.NATIVE;
+ public void markSkewedJoin() {
+ feature.set(OPER_FEATURE.SKEWEDJOIN.ordinal());
}
- public boolean isUnion() {
- return union;
+ public boolean isLimit() {
+ return feature.get(OPER_FEATURE.LIMIT.ordinal());
}
- public void setUnion() {
- union = true;
+ public void markLimit() {
+ feature.set(OPER_FEATURE.LIMIT.ordinal());
}
- public boolean isIndexer() {
- return (feature == OPER_FEATURE.INDEXER);
+ public boolean isLimitAfterSort() {
+ return feature.get(OPER_FEATURE.LIMIT_AFTER_SORT.ordinal());
}
- public void markIndexer() {
- feature = OPER_FEATURE.INDEXER;
+ public void markLimitAfterSort() {
+ feature.set(OPER_FEATURE.LIMIT_AFTER_SORT.ordinal());
}
- public boolean isSampler() {
- return (feature == OPER_FEATURE.SAMPLER);
- }
-
- public void markSampler() {
- feature = OPER_FEATURE.SAMPLER;
+ public boolean isUnion() {
+ return feature.get(OPER_FEATURE.UNION.ordinal());
}
- public boolean isSampleAggregation() {
- return (feature == OPER_FEATURE.SAMPLE_AGGREGATOR);
+ public void markUnion() {
+ feature.set(OPER_FEATURE.UNION.ordinal());
}
- public void markSampleAggregation() {
- feature = OPER_FEATURE.SAMPLE_AGGREGATOR;
- }
-
- public boolean isSampleBasedPartitioner() {
- return (feature == OPER_FEATURE.SAMPLE_BASED_PARTITIONER);
+ public boolean isNative() {
+ return feature.get(OPER_FEATURE.NATIVE.ordinal());
}
- public void markSampleBasedPartitioner() {
- feature = OPER_FEATURE.SAMPLE_BASED_PARTITIONER;
+ public void markNative() {
+ feature.set(OPER_FEATURE.NATIVE.ordinal());
}
public void setNeedEstimatedQuantile(boolean needEstimateParallelism) {
@@ -473,30 +503,6 @@ public class TezOperator extends Operato
return sortOrder;
}
- public void setGlobalSort(boolean globalSort) {
- this.globalSort = globalSort;
- }
-
- public boolean isGlobalSort() {
- return globalSort;
- }
-
- public void setLimitAfterSort(boolean limitAfterSort) {
- this.limitAfterSort = limitAfterSort;
- }
-
- public boolean isLimitAfterSort() {
- return limitAfterSort;
- }
-
- public void setSkewedJoin(boolean skewedJoin) {
- this.skewedJoin = skewedJoin;
- }
-
- public boolean isSkewedJoin() {
- return skewedJoin;
- }
-
protected void noCombineSmallSplits() {
combineSmallSplits = false;
}
Modified: pig/trunk/test/org/apache/pig/test/TestLimitVariable.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLimitVariable.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLimitVariable.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLimitVariable.java Thu Sep 18 14:16:39 2014
@@ -20,16 +20,18 @@ package org.apache.pig.test;
import java.io.File;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigImplConstants;
import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.newplan.logical.visitor.ScalarVariableValidator;
import org.junit.AfterClass;
-import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -116,6 +118,34 @@ public class TestLimitVariable {
Util.checkQueryOutputs(itE, expectedResE);
}
+ @Test
+ public void testLimitVariable4() throws IOException {
+ String query =
+ "a = load '" + inputFile.getName() + "' as (x:int, y:int);" +
+ "b = group a all;" +
+ "c = foreach b generate COUNT(a) as sum;" +
+ "d = order a by $0 DESC;" +
+ "e = filter d by $0 != 4;" +
+ "f = limit e c.sum/2;" // return top half of the tuples
+ ;
+
+ try {
+ HashSet<String> disabledOptimizerRules = new HashSet<String>();
+ disabledOptimizerRules.add("PushUpFilter");
+ pigServer.getPigContext().getProperties().setProperty(PigImplConstants.PIG_OPTIMIZER_RULES_KEY,
+ ObjectSerializer.serialize(disabledOptimizerRules));
+ Util.registerMultiLineQuery(pigServer, query);
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ // Even if push up filter is disabled order should be retained
+ List<Tuple> expectedRes = Util.getTuplesFromConstantTupleStrings(new String[] {
+ "(6,15)", "(5,10)", "(3,10)" });
+ Util.checkQueryOutputs(it, expectedRes);
+ } finally {
+ pigServer.getPigContext().getProperties().remove(PigImplConstants.PIG_OPTIMIZER_RULES_KEY);
+ }
+ }
+
@Test(expected=FrontendException.class)
public void testLimitVariableException1() throws Throwable {
String query =
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld?rev=1625989&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld Thu Sep 18 14:16:39 2014
@@ -0,0 +1,89 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-49
+#--------------------------------------------------
+Tez vertex scope-12 -> Tez vertex scope-33,Tez vertex scope-22,
+Tez vertex scope-22 -> Tez vertex scope-33,
+Tez vertex scope-33 -> Tez vertex scope-35,
+Tez vertex scope-35 -> Tez vertex scope-46,
+Tez vertex scope-46
+
+Tez vertex scope-12
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-15 -> scope-22
+| |
+| Constant(DummyVal) - scope-14
+|
+|---New For Each(false,false,true)[tuple] - scope-21
+ | |
+ | Project[int][0] - scope-8
+ | |
+ | Project[int][1] - scope-9
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-20
+ | |
+ | |---Project[tuple][*] - scope-19
+ |
+ |---ReservoirSample - scope-18
+ |
+ |---b: Local Rearrange[tuple]{tuple}(false) - scope-13 -> scope-33
+ | |
+ | Project[int][0] - scope-8
+ | |
+ | Project[int][1] - scope-9
+ |
+ |---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-22
+# Plan on vertex
+POValueOutputTez - scope-32 -> [scope-33]
+|
+|---New For Each(false)[tuple] - scope-31
+ | |
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-30
+ | |
+ | |---Project[tuple][*] - scope-29
+ |
+ |---New For Each(false,false)[tuple] - scope-28
+ | |
+ | Constant(-1) - scope-27
+ | |
+ | Project[bag][1] - scope-24
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-23
+Tez vertex scope-33
+# Plan on vertex
+POIdentityInOutTez - scope-34 <- scope-12 -> scope-35
+| |
+| Project[int][0] - scope-8
+| |
+| Project[int][1] - scope-9
+Tez vertex scope-35
+# Plan on vertex
+POValueOutputTez - scope-45 -> [scope-46]
+|
+|---Limit - scope-44
+ |
+ |---New For Each(true)[tuple] - scope-43
+ | |
+ | Project[bag][1] - scope-42
+ |
+ |---Package(LitePackager)[tuple]{tuple} - scope-41
+Tez vertex scope-46
+# Plan on vertex
+c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-11
+|
+|---Limit - scope-48
+ |
+ |---POValueInputTez - scope-47 <- scope-35
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld?rev=1625989&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld Thu Sep 18 14:16:39 2014
@@ -0,0 +1,151 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-71
+#--------------------------------------------------
+Tez vertex scope-33 -> Tez vertex scope-37,Tez vertex scope-60,Tez vertex scope-49,
+Tez vertex scope-49 -> Tez vertex scope-60,
+Tez vertex scope-60 -> Tez vertex scope-62,
+Tez vertex scope-37 -> Tez vertex scope-67,Tez vertex scope-62,
+Tez vertex scope-62 -> Tez vertex scope-67,
+Tez vertex scope-67
+
+Tez vertex scope-33
+# Plan on vertex
+a: Split - scope-92
+| |
+| g: Local Rearrange[tuple]{chararray}(false) - scope-83 -> scope-37
+| | |
+| | Project[chararray][0] - scope-85
+| |
+| |---h: New For Each(false,false)[bag] - scope-72
+| | |
+| | Project[chararray][0] - scope-73
+| | |
+| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-74
+| | |
+| | |---Project[bag][1] - scope-75
+| |
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-86
+| |
+| Local Rearrange[tuple]{tuple}(false) - scope-42 -> scope-49
+| | |
+| | Constant(DummyVal) - scope-41
+| |
+| |---New For Each(false,false,true)[tuple] - scope-48
+| | |
+| | Project[int][0] - scope-11
+| | |
+| | Project[int][1] - scope-12
+| | |
+| | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-47
+| | |
+| | |---Project[tuple][*] - scope-46
+| |
+| |---ReservoirSample - scope-45
+| |
+| |---b: Local Rearrange[tuple]{tuple}(false) - scope-40 -> scope-60
+| | |
+| | Project[int][0] - scope-11
+| | |
+| | Project[int][1] - scope-12
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-49
+# Plan on vertex
+POValueOutputTez - scope-59 -> [scope-60]
+|
+|---New For Each(false)[tuple] - scope-58
+ | |
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-57
+ | |
+ | |---Project[tuple][*] - scope-56
+ |
+ |---New For Each(false,false)[tuple] - scope-55
+ | |
+ | Constant(-1) - scope-54
+ | |
+ | Project[bag][1] - scope-51
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-50
+Tez vertex scope-60
+# Plan on vertex
+POIdentityInOutTez - scope-61 <- scope-33 -> scope-62
+| |
+| Project[int][0] - scope-11
+| |
+| Project[int][1] - scope-12
+Tez vertex scope-37
+# Combine plan on edge <scope-33>
+g: Local Rearrange[tuple]{chararray}(false) - scope-87 -> scope-37
+| |
+| Project[chararray][0] - scope-89
+|
+|---h: New For Each(false,false)[bag] - scope-76
+ | |
+ | Project[chararray][0] - scope-77
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-78
+ | |
+ | |---Project[bag][1] - scope-79
+ |
+ |---g: Package(CombinerPackager)[tuple]{chararray} - scope-82
+# Plan on vertex
+POValueOutputTez - scope-70 -> [scope-67, scope-62]
+|
+|---h: New For Each(false)[bag] - scope-23
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-21
+ | |
+ | |---Project[bag][1] - scope-80
+ |
+ |---g: Package(CombinerPackager)[tuple]{chararray} - scope-17
+Tez vertex scope-62
+# Plan on vertex
+POValueOutputTez - scope-66 -> [scope-67]
+|
+|---c: Limit - scope-25
+ | |
+ | Divide[long] - scope-31
+ | |
+ | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.ReadScalarsTez)[long] - scope-28
+ | | |
+ | | |---Constant(0) - scope-26
+ | |
+ | |---Cast[long] - scope-30
+ | |
+ | |---Constant(2) - scope-29
+ |
+ |---New For Each(true)[tuple] - scope-65
+ | |
+ | Project[bag][1] - scope-64
+ |
+ |---Package(LitePackager)[tuple]{tuple} - scope-63
+Tez vertex scope-67
+# Plan on vertex
+c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-32
+|
+|---c: Limit - scope-69
+ | |
+ | Divide[long] - scope-31
+ | |
+ | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.ReadScalarsTez)[long] - scope-28
+ | | |
+ | | |---Constant(0) - scope-26
+ | |
+ | |---Cast[long] - scope-30
+ | |
+ | |---Constant(2) - scope-29
+ |
+ |---POValueInputTez - scope-68 <- scope-62
Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1625989&r1=1625988&r2=1625989&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Thu Sep 18 14:16:39 2014
@@ -142,7 +142,31 @@ public class TestTezCompiler {
"c = foreach b generate y;" +
"store c into 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-1.gld");
+ }
+
+ @Test
+ public void testLimitOrderby() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = order a by x, y;" +
+ "c = limit b 10;" +
+ "store c into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld");
+ }
+
+ @Test
+ public void testLimitScalarOrderby() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = order a by x, y;" +
+ "g = group a all;" +
+ "h = foreach g generate COUNT(a) as sum;" +
+ "c = limit b h.sum/2;" +
+ "store c into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld");
}
@Test