You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/09/18 22:20:59 UTC
svn commit: r1626075 [2/2] - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/tez/
src/org/apache/pig/backend/hadoop/executionengine/tez/operators/
src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/
src/org/apache/pig/...
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/AccumulatorOptimizer.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/AccumulatorOptimizer.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/AccumulatorOptimizer.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/AccumulatorOptimizer.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/AccumulatorOptimizer.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/AccumulatorOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/AccumulatorOptimizer.java Thu Sep 18 20:20:56 2014
@@ -16,8 +16,11 @@
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
import org.apache.pig.backend.hadoop.executionengine.util.AccumulatorOptimizerUtil;
import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.VisitorException;
@@ -32,6 +35,7 @@ public class AccumulatorOptimizer extend
super(plan, new DepthFirstWalker<TezOperator, TezOperPlan>(plan));
}
+ @Override
public void visitTezOp(TezOperator tezOp) throws VisitorException {
AccumulatorOptimizerUtil.addAccumulator(tezOp.plan);
}
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/CombinerOptimizer.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/CombinerOptimizer.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/CombinerOptimizer.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/CombinerOptimizer.java Thu Sep 18 20:20:56 2014
@@ -15,13 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.util.List;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POLocalRearrangeTez;
import org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil;
import org.apache.pig.impl.plan.CompilationMessageCollector;
import org.apache.pig.impl.plan.DepthFirstWalker;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/LoaderProcessor.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/LoaderProcessor.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/LoaderProcessor.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/LoaderProcessor.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/LoaderProcessor.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/LoaderProcessor.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/LoaderProcessor.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez.optimizers;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.io.IOException;
import java.util.ArrayList;
@@ -33,10 +33,10 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
-import org.apache.pig.backend.hadoop.executionengine.tez.POSimpleTezLoad;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOpPlanVisitor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POSimpleTezLoad;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.plan.DependencyOrderWalker;
@@ -66,8 +66,8 @@ public class LoaderProcessor extends Tez
* @return true if any POLoads were found, else false.
* @throws VisitorException
* @throws IOException
- * @throws InterruptedException
- * @throws ClassNotFoundException
+ * @throws InterruptedException
+ * @throws ClassNotFoundException
*/
private List<POLoad> processLoads(TezOperator tezOp
) throws VisitorException, IOException, ClassNotFoundException, InterruptedException {
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/MultiQueryOptimizerTez.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/MultiQueryOptimizerTez.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/MultiQueryOptimizerTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.util.ArrayList;
import java.util.List;
@@ -26,6 +26,13 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.runtime.TezInput;
import org.apache.pig.backend.hadoop.executionengine.tez.util.TezCompilerUtil;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.PlanException;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/NoopFilterRemover.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/NoopFilterRemover.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/NoopFilterRemover.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/NoopFilterRemover.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/NoopFilterRemover.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/NoopFilterRemover.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/NoopFilterRemover.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez.optimizers;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.util.List;
@@ -26,9 +26,9 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOpPlanVisitor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
import org.apache.pig.impl.plan.DependencyOrderWalker;
import org.apache.pig.impl.plan.PlanException;
import org.apache.pig.impl.plan.VisitorException;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/ParallelismSetter.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/ParallelismSetter.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/ParallelismSetter.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/ParallelismSetter.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/ParallelismSetter.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/ParallelismSetter.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/ParallelismSetter.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez.optimizers;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.io.IOException;
import java.util.Map;
@@ -23,13 +23,11 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.PigConfiguration;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.backend.hadoop.executionengine.tez.NativeTezOper;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezEdgeDescriptor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOpPlanVisitor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperDependencyParallelismEstimator;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezParallelismEstimator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.NativeTezOper;
import org.apache.pig.backend.hadoop.executionengine.util.ParallelConstantVisitor;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.DependencyOrderWalker;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/SecondaryKeyOptimizerTez.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/SecondaryKeyOptimizerTez.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/SecondaryKeyOptimizerTez.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/SecondaryKeyOptimizerTez.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/SecondaryKeyOptimizerTez.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/SecondaryKeyOptimizerTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/SecondaryKeyOptimizerTez.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.util.List;
@@ -24,6 +24,11 @@ import org.apache.commons.logging.LogFac
import org.apache.pig.backend.hadoop.executionengine.optimizer.SecondaryKeyOptimizer;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POLocalRearrangeTez;
import org.apache.pig.backend.hadoop.executionengine.util.SecondaryKeyOptimizerUtil;
import org.apache.pig.backend.hadoop.executionengine.util.SecondaryKeyOptimizerUtil.SecondaryKeyOptimizerInfo;
import org.apache.pig.classification.InterfaceAudience;
@@ -52,7 +57,7 @@ public class SecondaryKeyOptimizerTez ex
}
// Current code does not handle more than one predecessors
- // even though it is possible. The problem is when we
+ // even though it is possible. The problem is when we
// process the first predecessor, we remove the foreach inner
// operators from the reduce side, and the second predecessor
// cannot see them
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezOperDependencyParallelismEstimator.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperDependencyParallelismEstimator.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezOperDependencyParallelismEstimator.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezOperDependencyParallelismEstimator.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperDependencyParallelismEstimator.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperDependencyParallelismEstimator.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezOperDependencyParallelismEstimator.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.io.IOException;
import java.util.LinkedList;
@@ -37,6 +37,11 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POLocalRearrangeTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez;
import org.apache.pig.backend.hadoop.executionengine.util.ParallelConstantVisitor;
import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.OperatorKey;
@@ -47,13 +52,13 @@ import org.apache.tez.dag.api.EdgeProper
* Estimate the parallelism of the vertex using:
* 1. parallelism of the predecessors
* 2. bloating factor of the physical plan of the predecessor
- *
+ *
* Since currently it is only possible to reduce the parallelism
* estimation is exaggerated and will rely on Tez runtime to
* descrease the parallelism
*/
public class TezOperDependencyParallelismEstimator implements TezParallelismEstimator {
-
+
static private int maxTaskCount;
static final double DEFAULT_FLATTEN_FACTOR = 10;
static final double DEFAULT_FILTER_FACTOR = 0.7;
@@ -61,14 +66,14 @@ public class TezOperDependencyParallelis
@Override
public int estimateParallelism(TezOperPlan plan, TezOperator tezOper, Configuration conf) throws IOException {
-
+
if (tezOper.isVertexGroup()) {
return -1;
}
-
+
maxTaskCount = conf.getInt(PigReducerEstimator.MAX_REDUCER_COUNT_PARAM,
PigReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);
-
+
// If parallelism is set explicitly, respect it
if (tezOper.getRequestedParallelism()!=-1) {
return tezOper.getRequestedParallelism();
@@ -96,7 +101,7 @@ public class TezOperDependencyParallelis
entry.getValue().dataMovementType==DataMovementType.ONE_TO_ONE) {
double predParallelism = pred.getEffectiveParallelism();
if (predParallelism==-1) {
- throw new IOException("Cannot estimate parallelism for " + tezOper.getOperatorKey().toString()
+ throw new IOException("Cannot estimate parallelism for " + tezOper.getOperatorKey().toString()
+ ", effective parallelism for predecessor " + tezOper.getOperatorKey().toString()
+ " is -1");
}
@@ -122,7 +127,7 @@ public class TezOperDependencyParallelis
}
}
sortOper = plan.getSuccessors(rangePartionerOper).get(0);
-
+
if (sortOper.getRequestedParallelism()!=-1) {
ParallelConstantVisitor visitor =
@@ -130,7 +135,7 @@ public class TezOperDependencyParallelis
visitor.visit();
}
}
-
+
return Math.min(roundedEstimatedParallelism, maxTaskCount);
}
@@ -143,7 +148,7 @@ public class TezOperDependencyParallelis
return plan.getOperator(unionPred);
}
}
-
+
}
else if (pred.getOperatorKey().toString().equals(inputKey)) {
return pred;
@@ -171,7 +176,7 @@ public class TezOperDependencyParallelis
}
factor *= DEFAULT_FILTER_FACTOR;
}
-
+
@Override
public void visitPOForEach(POForEach nfe) throws VisitorException {
List<Boolean> flattens = nfe.getToBeFlattened();
@@ -186,28 +191,30 @@ public class TezOperDependencyParallelis
factor *= DEFAULT_FLATTEN_FACTOR;
}
}
-
+
@Override
public void visitLimit(POLimit lim) throws VisitorException {
factor = DEFAULT_LIMIT_FACTOR;
}
-
+
+ @Override
public void visitFRJoin(POFRJoin join) throws VisitorException {
factor *= DEFAULT_FLATTEN_FACTOR;
}
+ @Override
public void visitMergeJoin(POMergeJoin join) throws VisitorException {
factor *= DEFAULT_FLATTEN_FACTOR;
}
@Override
public void visitPackage(POPackage pkg) throws VisitorException{
- // JoinPackager is equivalent to a foreach flatten after shuffle
+ // JoinPackager is equivalent to a foreach flatten after shuffle
if (pkg.getPkgr() instanceof JoinPackager) {
factor *= DEFAULT_FLATTEN_FACTOR;
}
}
-
+
@Override
public void visitSplit(POSplit sp) throws VisitorException {
// Find the split branch connecting to current operator
@@ -217,7 +224,7 @@ public class TezOperDependencyParallelis
visit();
popWalker();
}
-
+
private static PhysicalPlan getSplitBranch(POSplit split, String outputKey) throws VisitorException {
List<PhysicalPlan> plans = split.getPlans();
for (PhysicalPlan plan : plans) {
@@ -232,10 +239,10 @@ public class TezOperDependencyParallelis
}
return null;
}
-
+
public double getFactor() {
return factor;
}
-
+
}
}
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezParallelismEstimator.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezParallelismEstimator.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezParallelismEstimator.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezParallelismEstimator.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezParallelismEstimator.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezParallelismEstimator.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezParallelismEstimator.java Thu Sep 18 20:20:56 2014
@@ -15,11 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
public interface TezParallelismEstimator {
public int estimateParallelism(TezOperPlan plan, TezOperator tezOper, Configuration conf) throws IOException;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/UnionOptimizer.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/UnionOptimizer.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/UnionOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez.optimizers;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.util.ArrayList;
import java.util.LinkedList;
@@ -26,16 +26,16 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
-import org.apache.pig.backend.hadoop.executionengine.tez.POStoreTez;
-import org.apache.pig.backend.hadoop.executionengine.tez.POValueOutputTez;
-import org.apache.pig.backend.hadoop.executionengine.tez.RoundRobinPartitioner;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezEdgeDescriptor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezInput;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOpPlanVisitor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator.VertexGroupInfo;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOutput;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator.VertexGroupInfo;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POStoreTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.runtime.RoundRobinPartitioner;
+import org.apache.pig.backend.hadoop.executionengine.tez.runtime.TezInput;
+import org.apache.pig.backend.hadoop.executionengine.tez.runtime.TezOutput;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.ReverseDependencyOrderWalker;
import org.apache.pig.impl.plan.VisitorException;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/FindQuantilesTez.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/FindQuantilesTez.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/FindQuantilesTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.udf;
import java.io.IOException;
import java.util.Iterator;
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFac
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.InputSizeReducerEstimator;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
+import org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
@@ -37,15 +38,15 @@ public class FindQuantilesTez extends Fi
private static final Log LOG = LogFactory.getLog(FindQuantilesTez.class);
private static TupleFactory tf = TupleFactory.getInstance();
-
+
public FindQuantilesTez() {
super();
}
-
+
public FindQuantilesTez(String[] args) {
super(args);
}
-
+
@Override
public Map<String, Object> exec(Tuple in) throws IOException {
// In Tez, we also need to estimate the quantiles with regard to sample
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PartitionSkewedKeysTez.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PartitionSkewedKeysTez.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PartitionSkewedKeysTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.udf;
import java.io.IOException;
import java.util.Iterator;
@@ -25,6 +25,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.InputSizeReducerEstimator;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
+import org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.builtin.PartitionSkewedKeys;
@@ -45,7 +46,7 @@ public class PartitionSkewedKeysTez exte
if (in == null || in.size() == 0) {
return null;
}
-
+
int estimatedNumReducers = -1;
boolean estimate_sample_quantile = PigMapReduce.sJobConfInternal.get().getBoolean
(PigProcessor.ESTIMATE_PARALLELISM, false);
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/ReadScalarsTez.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/ReadScalarsTez.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/ReadScalarsTez.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/ReadScalarsTez.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/ReadScalarsTez.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/ReadScalarsTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/ReadScalarsTez.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.plan.udf;
import java.io.IOException;
import java.util.Map;
@@ -26,6 +26,8 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.EvalFunc;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.tez.runtime.ObjectCache;
+import org.apache.pig.backend.hadoop.executionengine.tez.runtime.TezInput;
import org.apache.pig.data.Tuple;
import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.library.api.KeyValueReader;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ObjectCache.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/ObjectCache.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ObjectCache.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ObjectCache.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/ObjectCache.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/ObjectCache.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/ObjectCache.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PartitionerDefinedVertexManager.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PartitionerDefinedVertexManager.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PartitionerDefinedVertexManager.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PartitionerDefinedVertexManager.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PartitionerDefinedVertexManager.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PartitionerDefinedVertexManager.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PartitionerDefinedVertexManager.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import java.util.HashMap;
import java.util.List;
@@ -35,7 +35,7 @@ import org.apache.tez.runtime.api.events
import com.google.common.collect.Lists;
/**
- * VertexManagerPlugin used by sorting job of order by and skewed join.
+ * VertexManagerPlugin used by sorting job of order by and skewed join.
* What is does is to set parallelism of the sorting vertex
* according to numParallelism specified by the predecessor vertex.
* The complex part is the PigOrderByEdgeManager, which specify how
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -44,6 +44,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez;
import org.apache.pig.data.SchemaTupleBackend;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/RoundRobinPartitioner.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/RoundRobinPartitioner.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/RoundRobinPartitioner.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/RoundRobinPartitioner.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/RoundRobinPartitioner.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/RoundRobinPartitioner.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/RoundRobinPartitioner.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Partitioner;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/SkewedPartitionerTez.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/SkewedPartitionerTez.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/SkewedPartitionerTez.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/SkewedPartitionerTez.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/SkewedPartitionerTez.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/SkewedPartitionerTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/SkewedPartitionerTez.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import java.util.Iterator;
import java.util.Map;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezInput.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezInput.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezInput.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezInput.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezInput.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezInput.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezInput.java Thu Sep 18 20:20:56 2014
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import java.util.Map;
import java.util.Set;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezOutput.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezOutput.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezOutput.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezOutput.java Thu Sep 18 20:20:56 2014
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import java.util.Map;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezTaskConfigurable.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezTaskConfigurable.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezTaskConfigurable.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezTaskConfigurable.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezTaskConfigurable.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezTaskConfigurable.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezTaskConfigurable.java Thu Sep 18 20:20:56 2014
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.tez.runtime.api.ProcessorContext;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezTaskContext.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezTaskContext.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezTaskContext.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezTaskContext.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezTaskContext.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezTaskContext.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/TezTaskContext.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.pig.backend.hadoop.executionengine.TaskContext;
Copied: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/WeightedRangePartitionerTez.java (from r1625992, pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/WeightedRangePartitionerTez.java)
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/WeightedRangePartitionerTez.java?p2=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/WeightedRangePartitionerTez.java&p1=pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/WeightedRangePartitionerTez.java&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/WeightedRangePartitionerTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/WeightedRangePartitionerTez.java Thu Sep 18 20:20:56 2014
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.backend.hadoop.executionengine.tez;
+package org.apache.pig.backend.hadoop.executionengine.tez.runtime;
import java.util.Map;
import java.util.Map.Entry;
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java Thu Sep 18 20:20:56 2014
@@ -12,13 +12,13 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
-import org.apache.pig.backend.hadoop.executionengine.tez.POLocalRearrangeTez;
-import org.apache.pig.backend.hadoop.executionengine.tez.POStoreTez;
-import org.apache.pig.backend.hadoop.executionengine.tez.POValueOutputTez;
-import org.apache.pig.backend.hadoop.executionengine.tez.RoundRobinPartitioner;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezEdgeDescriptor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POLocalRearrangeTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POStoreTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.runtime.RoundRobinPartitioner;
import org.apache.pig.data.DataType;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.PigContext;
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezScriptState.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezScriptState.java?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezScriptState.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezScriptState.java Thu Sep 18 20:20:56 2014
@@ -31,8 +31,8 @@ import org.apache.pig.LoadFunc;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.newplan.Operator;
import org.apache.pig.tools.pigstats.JobStats;
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezStats.java?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezStats.java Thu Sep 18 20:20:56 2014
@@ -28,15 +28,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobID;
import org.apache.pig.PigRunner.ReturnCode;
-import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NativeMapReduceOper;
-import org.apache.pig.backend.hadoop.executionengine.tez.NativeTezOper;
import org.apache.pig.backend.hadoop.executionengine.tez.TezExecType;
import org.apache.pig.backend.hadoop.executionengine.tez.TezJob;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOpPlanVisitor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.NativeTezOper;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.plan.DependencyOrderWalker;
@@ -44,7 +42,6 @@ import org.apache.pig.impl.plan.VisitorE
import org.apache.pig.tools.pigstats.InputStats;
import org.apache.pig.tools.pigstats.OutputStats;
import org.apache.pig.tools.pigstats.PigStats;
-import org.apache.pig.tools.pigstats.mapreduce.MRJobStats;
import org.apache.tez.common.TezUtils;
import org.apache.tez.common.counters.CounterGroup;
import org.apache.tez.common.counters.DAGCounter;
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld Thu Sep 18 20:20:56 2014
@@ -51,7 +51,7 @@ POValueOutputTez - scope-32 -> [scope-3
|
|---New For Each(false)[tuple] - scope-31
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-30
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-30
| |
| |---Project[tuple][*] - scope-29
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-3.gld Thu Sep 18 20:20:56 2014
@@ -68,7 +68,7 @@ POValueOutputTez - scope-59 -> [scope-6
|
|---New For Each(false)[tuple] - scope-58
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-57
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-57
| |
| |---Project[tuple][*] - scope-56
|
@@ -119,7 +119,7 @@ POValueOutputTez - scope-66 -> [scope-6
| |
| Divide[long] - scope-31
| |
- | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.ReadScalarsTez)[long] - scope-28
+ | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[long] - scope-28
| | |
| | |---Constant(0) - scope-26
| |
@@ -140,7 +140,7 @@ c: Store(file:///tmp/output:org.apache.p
| |
| Divide[long] - scope-31
| |
- | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.ReadScalarsTez)[long] - scope-28
+ | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[long] - scope-28
| | |
| | |---Constant(0) - scope-26
| |
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Thu Sep 18 20:20:56 2014
@@ -254,7 +254,7 @@ POValueOutputTez - scope-369 -> [scope-
|
|---New For Each(false)[tuple] - scope-368
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-367
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-367
| |
| |---Project[tuple][*] - scope-366
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Thu Sep 18 20:20:56 2014
@@ -150,7 +150,7 @@ POValueOutputTez - scope-157 -> [scope-
|
|---New For Each(false)[tuple] - scope-156
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-155
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-155
| |
| |---Project[tuple][*] - scope-154
|
Copied: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld (from r1625992, pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld)
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld?p2=pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld&p1=pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld Thu Sep 18 20:20:56 2014
@@ -46,7 +46,7 @@ POValueOutputTez - scope-29 -> [scope-3
|
|---New For Each(false)[tuple] - scope-28
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-27
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-27
| |
| |---Project[tuple][*] - scope-26
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld Thu Sep 18 20:20:56 2014
@@ -62,7 +62,7 @@ POValueOutputTez - scope-42 -> [scope-4
|
|---New For Each(false)[tuple] - scope-41
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-40
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-40
| |
| |---Project[tuple][*] - scope-39
|
Copied: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld (from r1625992, pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld)
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld?p2=pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld&p1=pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld&r1=1625992&r2=1626075&rev=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld Thu Sep 18 20:20:56 2014
@@ -47,7 +47,7 @@ POValueOutputTez - scope-45 -> [scope-4
|
|---New For Each(false)[tuple] - scope-44
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.PartitionSkewedKeysTez)[tuple] - scope-43
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-43
| |
| |---Project[tuple][*] - scope-42
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld Thu Sep 18 20:20:56 2014
@@ -69,7 +69,7 @@ POValueOutputTez - scope-138 -> [scope-
|
|---New For Each(false)[tuple] - scope-137
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.PartitionSkewedKeysTez)[tuple] - scope-136
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-136
| |
| |---Project[tuple][*] - scope-135
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld Thu Sep 18 20:20:56 2014
@@ -83,7 +83,7 @@ POValueOutputTez - scope-52 -> [scope-5
|
|---New For Each(false)[tuple] - scope-51
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.PartitionSkewedKeysTez)[tuple] - scope-50
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-50
| |
| |---Project[tuple][*] - scope-49
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld Thu Sep 18 20:20:56 2014
@@ -68,7 +68,7 @@ POValueOutputTez - scope-117 -> [scope-
|
|---New For Each(false)[tuple] - scope-116
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-115
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-115
| |
| |---Project[tuple][*] - scope-114
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld Thu Sep 18 20:20:56 2014
@@ -82,7 +82,7 @@ POValueOutputTez - scope-43 -> [scope-4
|
|---New For Each(false)[tuple] - scope-42
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-41
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-41
| |
| |---Project[tuple][*] - scope-40
|
Modified: pig/trunk/test/org/apache/pig/tez/TestGroupConstParallelTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestGroupConstParallelTez.java?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestGroupConstParallelTez.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestGroupConstParallelTez.java Thu Sep 18 20:20:56 2014
@@ -21,11 +21,11 @@ package org.apache.pig.tez;
import static org.junit.Assert.assertEquals;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezCompiler;
import org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.optimizers.LoaderProcessor;
-import org.apache.pig.backend.hadoop.executionengine.tez.optimizers.ParallelismSetter;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.LoaderProcessor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.ParallelismSetter;
import org.apache.pig.impl.PigContext;
import org.apache.pig.test.TestGroupConstParallel;
import org.apache.pig.tools.pigstats.PigStats.JobGraph;
Modified: pig/trunk/test/org/apache/pig/tez/TestJobSubmissionTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestJobSubmissionTez.java?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestJobSubmissionTez.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestJobSubmissionTez.java Thu Sep 18 20:20:56 2014
@@ -23,11 +23,11 @@ import static org.junit.Assert.assertTru
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezCompiler;
import org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.optimizers.LoaderProcessor;
-import org.apache.pig.backend.hadoop.executionengine.tez.optimizers.ParallelismSetter;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.LoaderProcessor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.ParallelismSetter;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.test.TestJobSubmission;
@@ -69,7 +69,7 @@ public class TestJobSubmissionTez extend
@Override
public void checkDefaultParallelResult(PhysicalPlan pp, PigContext pc) throws Exception {
TezOperPlan tezPlan = buildTezPlan(pp, pc);
-
+
LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
loaderStorer.visit();
Modified: pig/trunk/test/org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java Thu Sep 18 20:20:56 2014
@@ -22,9 +22,9 @@ import java.util.List;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezPOUserFuncVisitor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezPlanContainer;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPOUserFuncVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPlanContainer;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.test.TestLoaderStorerShipCacheFiles;
Modified: pig/trunk/test/org/apache/pig/tez/TestSecondarySortTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestSecondarySortTez.java?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestSecondarySortTez.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestSecondarySortTez.java Thu Sep 18 20:20:56 2014
@@ -20,10 +20,10 @@ package org.apache.pig.tez;
import org.apache.pig.PigConfiguration;
import org.apache.pig.backend.hadoop.executionengine.optimizer.SecondaryKeyOptimizer;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.CombinerOptimizer;
-import org.apache.pig.backend.hadoop.executionengine.tez.SecondaryKeyOptimizerTez;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezCompiler;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.CombinerOptimizer;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.SecondaryKeyOptimizerTez;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.test.MiniGenericCluster;
import org.apache.pig.test.TestSecondarySort;
Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Thu Sep 18 20:20:56 2014
@@ -32,10 +32,10 @@ import org.apache.pig.backend.executione
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher;
import org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezPlanContainer;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezPlanContainerNode;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezPlanContainerPrinter;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPlanContainer;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPlanContainerNode;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPlanContainerPrinter;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.NodeIdGenerator;
import org.apache.pig.impl.plan.OperatorKey;
@@ -96,7 +96,7 @@ public class TestTezCompiler {
"c = foreach b generate y;" +
"store c into 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC1.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Filter-1.gld");
}
@Test
@@ -107,7 +107,7 @@ public class TestTezCompiler {
"c = foreach b generate group, COUNT(a.x);" +
"store c into 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Group-1.gld");
}
@Test
@@ -119,7 +119,7 @@ public class TestTezCompiler {
"d = foreach c generate a::x as x, y, z;" +
"store d into 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC3.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Join-1.gld");
}
@Test
@@ -131,7 +131,7 @@ public class TestTezCompiler {
"d = foreach c generate a::x as x, y, z;" +
"store d into 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld");
}
@Test
@@ -177,7 +177,7 @@ public class TestTezCompiler {
"c = foreach b generate y;" +
"store c into 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC5.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-1.gld");
}
@Test
@@ -188,7 +188,7 @@ public class TestTezCompiler {
"c = foreach b { d = distinct a; generate COUNT(d); };" +
"store c into 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC13.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Distinct-2.gld");
}
@@ -202,7 +202,7 @@ public class TestTezCompiler {
"d = join a by x, b by x, c by x using 'replicated';" +
"store d into 'file:///tmp/output/d';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC10.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-1.gld");
}
@Test
@@ -215,7 +215,7 @@ public class TestTezCompiler {
"d = join b1 by group, c by x using 'replicated';" +
"store d into 'file:///tmp/output/e';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC11.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-FRJoin-2.gld");
}
@Test
@@ -225,7 +225,7 @@ public class TestTezCompiler {
"b = stream a through `stream.pl -n 5`;" +
"STORE b INTO 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC12.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Stream-1.gld");
}
@Test
@@ -236,11 +236,11 @@ public class TestTezCompiler {
"c = foreach b { d = limit a 10; e = order d by $1; f = order e by $0; generate group, f;};"+
"store c INTO 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC14.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SecKeySort-1.gld");
// With optimization turned off
setProperty(PigConfiguration.PIG_EXEC_NO_SECONDARY_KEY, "true");
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC15.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SecKeySort-2.gld");
}
@Test
@@ -250,7 +250,7 @@ public class TestTezCompiler {
"b = order a by x;" +
"STORE b INTO 'file:///tmp/output';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld");
}
// PIG-3759, PIG-3781
@@ -264,7 +264,7 @@ public class TestTezCompiler {
"d = foreach c generate group, COUNT(a.y), COUNT(b.z);" +
"store d into 'file:///tmp/output/d';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cogroup-1.gld");
}
@Test
Modified: pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java?rev=1626075&r1=1626074&r2=1626075&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java Thu Sep 18 20:20:56 2014
@@ -41,14 +41,14 @@ import org.apache.pig.PigConfiguration;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.MultiQueryOptimizerTez;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezCompiler;
import org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler;
import org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
-import org.apache.pig.backend.hadoop.executionengine.tez.optimizers.LoaderProcessor;
-import org.apache.pig.backend.hadoop.executionengine.tez.optimizers.ParallelismSetter;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.LoaderProcessor;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.MultiQueryOptimizerTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer.ParallelismSetter;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.util.Pair;
@@ -254,7 +254,7 @@ public class TestTezJobControlCompiler {
Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
assertEquals(leafVertex.getParallelism(), 15);
}
-
+
@Test
public void testTezParallelismEstimatorSplitBranch() throws Exception{
pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
@@ -270,7 +270,7 @@ public class TestTezJobControlCompiler {
Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
assertEquals(leafVertex.getParallelism(), 7);
}
-
+
@Test
public void testTezParallelismDefaultParallelism() throws Exception{
pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");