You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2015/05/22 02:06:17 UTC
svn commit: r1680969 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/
src/org/apache/pig/backend/hadoop/executioneng...
Author: daijy
Date: Fri May 22 00:06:16 2015
New Revision: 1680969
URL: http://svn.apache.org/r1680969
Log:
Rollback PIG-4066: An optimization for ROLLUP operation in Pig
Removed:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/RollupHIIPartitioner.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PORollupHIIForEach.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LORollupHIIForEach.java
pig/trunk/src/org/apache/pig/newplan/logical/rules/RollupHIIOptimizer.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/Main.java
pig/trunk/src/org/apache/pig/PigConstants.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java
pig/trunk/src/org/apache/pig/builtin/RollupDimensions.java
pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCube.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java
pig/trunk/src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java
pig/trunk/src/org/apache/pig/parser/AliasMasker.g
pig/trunk/src/org/apache/pig/parser/AstPrinter.g
pig/trunk/src/org/apache/pig/parser/AstValidator.g
pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g
pig/trunk/src/org/apache/pig/parser/QueryLexer.g
pig/trunk/src/org/apache/pig/parser/QueryParser.g
pig/trunk/test/org/apache/pig/test/TestCubeOperator.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 22 00:06:16 2015
@@ -72,8 +72,6 @@ PIG-4360: HBaseStorage should support se
PIG-4337: Split Types and MultiQuery e2e tests into multiple groups (rohini)
-PIG-4066: An optimization for ROLLUP operation in Pig (hxquangnhat via cheolsoo)
-
PIG-4333: Split BigData tests into multiple groups (rohini)
BUG FIXES
Modified: pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Fri May 22 00:06:16 2015
@@ -916,7 +916,6 @@ public class Main {
System.out.println(" GroupByConstParallelSetter - Force parallel 1 for \"group all\" statement");
System.out.println(" PartitionFilterOptimizer - Pushdown partition filter conditions to loader implementing LoadMetaData");
System.out.println(" PredicatePushdownOptimizer - Pushdown filter predicates to loader implementing LoadPredicatePushDown");
- System.out.println(" RollupHIIOptimizer - Apply Rollup HII optimization");
System.out.println(" All - Disable all optimizations");
System.out.println(" All optimizations listed here are enabled by default. Optimization values are case insensitive.");
System.out.println(" -v, -verbose - Print all error messages to screen");
Modified: pig/trunk/src/org/apache/pig/PigConstants.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConstants.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConstants.java (original)
+++ pig/trunk/src/org/apache/pig/PigConstants.java Fri May 22 00:06:16 2015
@@ -59,45 +59,4 @@ public class PigConstants {
public static final String TIME_UDFS_ELAPSED_TIME_COUNTER = "approx_microsecs";
public static final String TASK_INDEX = "mapreduce.task.index";
-
- /**
- * This parameter is used to check if the rollup is optimizable or not after going
- * through the RollupHIIOptimizer
- */
- public static final String PIG_HII_ROLLUP_OPTIMIZABLE = "pig.hii.rollup.optimizable";
-
- /**
- * This parameter stores the value of the pivot position. If the rollup is not optimizable
- * this value will be -1; If the rollup is optimizable: if the user did specify the pivot
- * in the rollup clause, this parameter will get that value; if the user did not specify
- * the pivot in the rollup clause, this parameter will get the value of the median position
- * of the fields in the rollup clause
- */
- public static final String PIG_HII_ROLLUP_PIVOT = "pig.hii.rollup.pivot";
-
- /**
- * This parameter stores the index of the first field involves in the rollup (or the first field
- * involves in the rollup after changing the position of rollup to the end in case of having cube)
- */
- public static final String PIG_HII_ROLLUP_FIELD_INDEX = "pig.hii.rollup.field.index";
-
- /**
- * This parameter stores the index of the first field involves in the rollup before
- * changing the position of rollup to the end in case of having cube
- */
- public static final String PIG_HII_ROLLUP_OLD_FIELD_INDEX = "pig.hii.rollup.old.field.index";
-
- /**
- * This parameter stores the size of total fields which involve in the CUBE clause. For example, we
- * have two CUBE clause:
- * B = CUBE A BY CUBE(year, month, day), ROLLUP(hour, minute, second);
- * B = CUBE A BY ROLLUP(year, month, day, hour, minute, second);
- * So this parameter will be 6 at both cases.
- */
- public static final String PIG_HII_NUMBER_TOTAL_FIELD = "pig.hii.number.total.field";
-
- /**
- * This parameter stores the number of algebraic functions that used after rollup.
- */
- public static final String PIG_HII_NUMBER_ALGEBRAIC = "pig.hii.number.algebraic";
}
\ No newline at end of file
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri May 22 00:06:16 2015
@@ -66,14 +66,12 @@ import org.apache.pig.FuncSpec;
import org.apache.pig.LoadFunc;
import org.apache.pig.OverwritableStoreFunc;
import org.apache.pig.PigConfiguration;
-import org.apache.pig.PigConstants;
import org.apache.pig.PigException;
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.HDataType;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
-import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.RollupHIIPartitioner;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.SecondaryKeyPartitioner;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.SkewedPartitioner;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.WeightedRangePartitioner;
@@ -185,8 +183,6 @@ public class JobControlCompiler{
public static final String PIG_MAP_STORES = "pig.map.stores";
public static final String PIG_REDUCE_STORES = "pig.reduce.stores";
- private static final String ROLLUP_PARTITIONER = RollupHIIPartitioner.class.getName();
-
// A mapping of job to pair of store locations and tmp locations for that job
private Map<Job, Pair<List<POStore>, Path>> jobStoreMap;
@@ -525,9 +521,6 @@ public class JobControlCompiler{
configureCompression(conf);
try{
- //Set default value for PIG_HII_ROLLUP_OPTIMIZABLE to false
- conf.setBoolean(PigConstants.PIG_HII_ROLLUP_OPTIMIZABLE, false);
-
//Process the POLoads
List<POLoad> lds = PlanHelper.getPhysicalOperators(mro.mapPlan, POLoad.class);
@@ -846,51 +839,14 @@ public class JobControlCompiler{
}
pack = (POPackage)mro.reducePlan.getRoots().get(0);
- if(pack!=null) {
- if(pack.getPivot()!=-1) {
- //Set value for PIG_HII_ROLLUP_OPTIMIZABLE to true
- conf.setBoolean(PigConstants.PIG_HII_ROLLUP_OPTIMIZABLE, true);
- //Set the pivot value
- conf.setInt(PigConstants.PIG_HII_ROLLUP_PIVOT, pack.getPivot());
- //Set the index of the first field involves in ROLLUP
- conf.setInt(PigConstants.PIG_HII_ROLLUP_FIELD_INDEX, pack.getRollupFieldIndex());
- //Set the original index of the first field involves in ROLLUP in case it was moved to the end
- //(if we have the combination of cube and rollup)
- conf.setInt(PigConstants.PIG_HII_ROLLUP_OLD_FIELD_INDEX, pack.getRollupOldFieldIndex());
- //Set the size of total fields that involve in CUBE clause
- conf.setInt(PigConstants.PIG_HII_NUMBER_TOTAL_FIELD, pack.getDimensionSize());
- //Set number of algebraic functions that used after rollup
- conf.setInt(PigConstants.PIG_HII_NUMBER_ALGEBRAIC, pack.getNumberAlgebraic());
- //Set number of reducer to 1 due to using IRG algorithm
- if(pack.getPivot() == 0 && !mro.reducePlan.isEmpty()) {
- updateNumReducers(plan, mro, nwJob);
- }
- }
- }
-
if (!pigContext.inIllustrator) {
mro.reducePlan.remove(pack);
}
-
- if (pack!=null && pack.getPivot()!=-1) {
- nwJob.setMapperClass(PigMapReduce.MapRollupHII.class);
- } else {
- nwJob.setMapperClass(PigMapReduce.Map.class);
- }
-
+ nwJob.setMapperClass(PigMapReduce.Map.class);
nwJob.setReducerClass(PigMapReduce.Reduce.class);
- // Set Rollup Partitioner in case the pivot is not equal to -1
- // and the custormPartitioner name is our rollup partitioner.
- if (mro.customPartitioner != null) {
- if (mro.customPartitioner.equals(ROLLUP_PARTITIONER)) {
- if (pack.getPivot()!=-1) {
- nwJob.setPartitionerClass(PigContext.resolveClassName(mro.customPartitioner));
- }
- } else {
- nwJob.setPartitionerClass(PigContext.resolveClassName(mro.customPartitioner));
- }
- }
+ if (mro.customPartitioner != null)
+ nwJob.setPartitionerClass(PigContext.resolveClassName(mro.customPartitioner));
if(!pigContext.inIllustrator)
conf.set("pig.mapPlan", ObjectSerializer.serialize(mro.mapPlan));
@@ -1106,26 +1062,6 @@ public class JobControlCompiler{
}
/**
- * If pivot position is zero, we use only one reducer
- * @param plan the MR plan
- * @param mro the MR operator
- * @param nwJob the current job
- * @throws IOException
- */
- public void updateNumReducers(MROperPlan plan, MapReduceOper mro,
- org.apache.hadoop.mapreduce.Job nwJob) throws IOException {
- // Change number of reducer to 1 if only IRG is used
- if (mro.customPartitioner != null && mro.customPartitioner.equals(ROLLUP_PARTITIONER)) {
- log.info("Changing Parallelism to 1 due to using IRG");
- }
- conf.setInt("pig.info.reducers.default.parallel", 1);
- conf.setInt("pig.info.reducers.requested.parallel", 1);
- conf.setInt("pig.info.reducers.estimated.parallel", 1);
- conf.setInt(MRConfiguration.REDUCE_TASKS, 1);
- nwJob.setNumReduceTasks(1);
- }
-
- /**
* Calculate the runtime #reducers based on the default_parallel, requested parallel and estimated
* parallel, and save it to MapReduceOper's runtimeParallelism.
* @return the runtimeParallelism
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Fri May 22 00:06:16 2015
@@ -76,7 +76,6 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartitionRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORollupHIIForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
@@ -1100,11 +1099,6 @@ public class MRCompiler extends PhyPlanV
}
@Override
- public void visitPORollupHIIForEach(PORollupHIIForEach op) throws VisitorException {
- visitPOForEach(op);
- }
-
- @Override
public void visitGlobalRearrange(POGlobalRearrange op) throws VisitorException{
try{
blocking(op);
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java Fri May 22 00:06:16 2015
@@ -30,7 +30,6 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.pig.JVMReuseManager;
import org.apache.pig.PigConstants;
import org.apache.pig.PigException;
@@ -51,7 +50,6 @@ import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.SchemaTupleBackend;
import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.NullablePartitionWritable;
import org.apache.pig.impl.io.NullableTuple;
@@ -139,92 +137,6 @@ public class PigGenericMapReduce {
}
/**
- * This map is only used for the Rollup when the RollupHIIOptimizer is enabled
- *
- */
- public static class MapRollupHII extends PigMapBase {
- @Override
- public void collect(Context oc, Tuple tuple)
- throws InterruptedException, IOException {
-
- Byte index = (Byte)tuple.get(0);
- PigNullableWritable key =
- HDataType.getWritableComparableTypes(tuple.get(1), keyType);
- NullableTuple val = new NullableTuple((Tuple)tuple.get(2));
-
- // Both the key and the value need the index. The key needs it so
- // that it can be sorted on the index in addition to the key
- // value. The value needs it so that POPackage can properly
- // assign the tuple to its slot in the projection.
- key.setIndex(index);
- val.setIndex(index);
-
- oc.write(key, val);
- }
-
- @Override
- public void cleanup(Context oc)
- throws InterruptedException, IOException {
-
- Configuration jConf = oc.getConfiguration();
-
- boolean isHII = jConf.getBoolean(PigConstants.PIG_HII_ROLLUP_OPTIMIZABLE, false);
- //If our rule is enabled and is using, there will be a PORollupHIIForEach
- //We will create marker tuples which are considered as markers for reducers
- //to calculate the remaining results when that reducer goes to the end of the
- //input records. This marker tuple will have larger size than the defaut by one
- //dimension. This addition dimension will be the value which are ranged from 0 to
- //number of reducers. By this addition, we can make sure that every reducers can
- //receive these marker tuples to finish their works.
- if(isHII) {
- int reducerNo = jConf.getInt("mapred.reduce.tasks", 0);
- int length = jConf.getInt(PigConstants.PIG_HII_NUMBER_TOTAL_FIELD, 0);
- int nAlgebraic = jConf.getInt(PigConstants.PIG_HII_NUMBER_ALGEBRAIC, 1);
-
- if(length == 0)
- return;
-
- TupleFactory mTupleFactory = TupleFactory.getInstance();
- //An array of marker tuples which has size equals to number of reducers
- Tuple group[] = new Tuple[reducerNo];
- int count = 0;
- //Make sure that all reducers will receive those marker tuples
- while(count < reducerNo) {
- //Create marker tuple with last field is the reducer's index,
- //the rest are null.
- group[count] = mTupleFactory.newTuple();
- for (int k = 0; k <= length; k++) {
- if(k < length) {
- group[count].append(null);
- } else {
- group[count].append(count);
- }
- }
-
- Tuple value = mTupleFactory.newTuple();
- Tuple []tmp = new Tuple[nAlgebraic];
- long valtmp = 1;
- for(int i = 0; i < nAlgebraic; i++){
- tmp[i] = mTupleFactory.newTuple();
- tmp[i].append(valtmp);
- value.append(tmp[i]);
- }
- Tuple out = mTupleFactory.newTuple();
- out.append(0);
- out.append(group[count]);
- out.append(value);
-
- PigNullableWritable key = HDataType.getWritableComparableTypes(out.get(1), keyType);
- NullableTuple val = new NullableTuple((Tuple)out.get(2));
- oc.write(key, val);
- count++;
- }
- }
- super.cleanup(oc);
- }
- }
-
- /**
* This "specialized" map class is ONLY to be used in pig queries with
* order by a udf. A UDF used for comparison in the order by expects
* to be handed tuples. Hence this map class ensures that the "key" used
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java Fri May 22 00:06:16 2015
@@ -45,7 +45,6 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.MonitoredUDFExecutor;
import org.apache.pig.builtin.MonitoredUDF;
-import org.apache.pig.builtin.RollupDimensions;
import org.apache.pig.data.DataType;
import org.apache.pig.data.SchemaTupleClassGenerator.GenContext;
import org.apache.pig.data.SchemaTupleFactory;
@@ -87,28 +86,6 @@ public class POUserFunc extends Expressi
private long timingFrequency = 100L;
private boolean doTiming = false;
- private static final String ROLLUP_UDF = RollupDimensions.class.getName();
- //the pivot value
- private int pivot = -1;
-
- private boolean rollupHIIoptimizable = false;
-
- public void setPivot(int pvt) {
- this.pivot = pvt;
- }
-
- public int getPivot() {
- return this.pivot;
- }
-
- public void setRollupHIIOptimizable(boolean check) {
- this.rollupHIIoptimizable = check;
- }
-
- public boolean getRollupHIIOptimizable() {
- return this.rollupHIIoptimizable;
- }
-
public PhysicalOperator getReferencedOperator() {
return referencedOperator;
}
@@ -150,17 +127,6 @@ public class POUserFunc extends Expressi
if (func.getClass().isAnnotationPresent(MonitoredUDF.class)) {
executor = new MonitoredUDFExecutor(func);
}
-
- if (funcSpec.getClassName().equals(ROLLUP_UDF) && this.rollupHIIoptimizable != false) {
- try {
- ((RollupDimensions) func).setPivot(this.pivot);
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- ((RollupDimensions) func).setRollupHIIOptimizable(this.rollupHIIoptimizable);
- }
-
//the next couple of initializations do not work as intended for the following reasons
//the reporter and pigLogger are member variables of PhysicalOperator
//when instanitateFunc is invoked at deserialization time, both
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java Fri May 22 00:06:16 2015
@@ -67,7 +67,6 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPreCombinerLocalRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POReservoirSample;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORollupHIIForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
@@ -153,10 +152,6 @@ public class PhyPlanVisitor extends Plan
}
}
- public void visitPORollupHIIForEach(PORollupHIIForEach nhfe) throws VisitorException {
- visitPOForEach(nhfe);
- }
-
public void visitUnion(POUnion un) throws VisitorException{
//do nothing
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java Fri May 22 00:06:16 2015
@@ -87,58 +87,6 @@ public class POPackage extends PhysicalO
private transient boolean useDefaultBag;
private transient int accumulativeBatchSize;
- //the pivot value
- private int pivot = -1;
- //the index of the first field involves in ROLLUP
- protected int rollupFieldIndex = 0;
- //the original index of the first field involves in ROLLUP in case it was moved to the end
- //(if we have the combination of cube and rollup)
- private int rollupOldFieldIndex = 0;
- //the size of total fields that involve in CUBE clause
- private int dimensionSize = 0;
- //number of algebraic function that used after rollup
- private int nAlgebraic = 0;
-
- public void setPivot(int pvt) {
- this.pivot = pvt;
- }
-
- public int getPivot() {
- return this.pivot;
- }
-
- public void setDimensionSize(int ds) {
- this.dimensionSize = ds;
- }
-
- public int getDimensionSize() {
- return this.dimensionSize;
- }
-
- public void setNumberAlgebraic(int na) {
- this.nAlgebraic = na;
- }
-
- public int getNumberAlgebraic() {
- return this.nAlgebraic;
- }
-
- public void setRollupOldFieldIndex(int rofi) {
- this.rollupOldFieldIndex = rofi;
- }
-
- public int getRollupOldFieldIndex() {
- return this.rollupOldFieldIndex;
- }
-
- public void setRollupFieldIndex(int rfi) {
- this.rollupFieldIndex = rfi;
- }
-
- public int getRollupFieldIndex() {
- return this.rollupFieldIndex;
- }
-
public POPackage(OperatorKey k) {
this(k, -1, null);
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java Fri May 22 00:06:16 2015
@@ -72,7 +72,6 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPoissonSample;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPreCombinerLocalRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POReservoirSample;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORollupHIIForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
@@ -237,12 +236,6 @@ public class PlanHelper {
}
@Override
- public void visitPORollupHIIForEach(PORollupHIIForEach hfe) throws VisitorException {
- super.visitPORollupHIIForEach(hfe);
- visit(hfe);
- }
-
- @Override
public void visitUnion(POUnion un) throws VisitorException {
super.visitUnion(un);
visit(un);
Modified: pig/trunk/src/org/apache/pig/builtin/RollupDimensions.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/RollupDimensions.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/RollupDimensions.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/RollupDimensions.java Fri May 22 00:06:16 2015
@@ -47,10 +47,6 @@ public class RollupDimensions extends Ev
private static BagFactory bf = BagFactory.getInstance();
private static TupleFactory tf = TupleFactory.getInstance();
private final String allMarker;
- // the pivot position
- private int pivot = -1;
- // to check if rollup is optimized or not
- private boolean rollupHIIoptimizable = false;
public RollupDimensions() {
this(null);
@@ -61,18 +57,6 @@ public class RollupDimensions extends Ev
this.allMarker = allMarker;
}
- public void setRollupHIIOptimizable(boolean check) {
- this.rollupHIIoptimizable = check;
- }
-
- public boolean getRollupHIIOptimizable() {
- return this.rollupHIIoptimizable;
- }
-
- public void setPivot(int pvt) throws IOException {
- this.pivot = pvt;
- }
-
@Override
public DataBag exec(Tuple tuple) throws IOException {
List<Tuple> result = Lists.newArrayListWithCapacity(tuple.size() + 1);
@@ -82,32 +66,12 @@ public class RollupDimensions extends Ev
return bf.newDefaultBag(result);
}
- private void iterativelyRollup(List<Tuple> result, Tuple input)
- throws IOException {
-
- Tuple tempTup = tf.newTuple(input.getAll());
-
- //if (this.rollupHIIoptimizable != null) { // rule is enabled
- if (this.rollupHIIoptimizable == true) {
- if (this.pivot == -1) // user did not specify the pivot position
- // --> IRG approach
- return;
- else { // user did specify the pivot position --> IRG + IRG
- if (this.pivot == 0) // we use the IRG approach
- return;
- else { // we use IRG+IRG approach
- for (int i = this.pivot - 1; i < input.size(); i++)
- tempTup.set(i, allMarker);
- result.add(tf.newTuple(tempTup.getAll()));
- }
- }
- }
- else { // we can not optimize --> Vanilla approach
- for (int i = input.size() - 1; i >= 0; i--) {
- tempTup.set(i, allMarker);
- result.add(tf.newTuple(tempTup.getAll()));
- }
- }
+ private void iterativelyRollup(List<Tuple> result, Tuple input) throws ExecException {
+ Tuple tempTup = tf.newTuple(input.getAll());
+ for (int i = input.size() - 1; i >= 0; i--) {
+ tempTup.set(i, allMarker);
+ result.add(tf.newTuple(tempTup.getAll()));
+ }
}
@Override
Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java Fri May 22 00:06:16 2015
@@ -60,7 +60,6 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.Subtract;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.builtin.RollupDimensions;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -502,28 +501,11 @@ public class ExpToPhyTranslationVisitor
public void visit( UserFuncExpression op ) throws FrontendException {
Object f = PigContext.instantiateFuncFromSpec(op.getFuncSpec());
PhysicalOperator p;
- String ROLLUP_UDF = RollupDimensions.class.getName();
if (f instanceof EvalFunc) {
p = new POUserFunc(new OperatorKey(DEFAULT_SCOPE, nodeGen
.getNextNodeId(DEFAULT_SCOPE)), -1,
null, op.getFuncSpec(), (EvalFunc) f);
((POUserFunc)p).setSignature(op.getSignature());
- if( op.getFuncSpec().toString().equals(ROLLUP_UDF)) {
- //Set the pivot value
- ((POUserFunc)p).setPivot(op.getPivot());
- if(op.getRollupHIIOptimizable()!=false) {
- ((POUserFunc)p).setRollupHIIOptimizable(true);
- //Set value for RollupHIIOptimizable and pivot of RollupDimension
- EvalFunc<?> tmp = ((POUserFunc)p).getFunc();
- ((RollupDimensions)tmp).setRollupHIIOptimizable(true);
- try {
- ((RollupDimensions)tmp).setPivot(op.getPivot());
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- }
//reinitialize input schema from signature
if (((POUserFunc)p).getFunc().getInputSchema() == null) {
((POUserFunc)p).setFuncInputSchema(op.getSignature());
Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java Fri May 22 00:06:16 2015
@@ -58,26 +58,6 @@ public class UserFuncExpression extends
private static int sigSeq=0;
private boolean viaDefine=false; //this represents whether the function was instantiate via a DEFINE statement or not
- private boolean rollupHIIoptimizable = false;
- //the pivot value
- private int pivot = -1;
-
- public void setPivot(int pvt) {
- this.pivot = pvt;
- }
-
- public int getPivot() {
- return this.pivot;
- }
-
- public void setRollupHIIOptimizable(boolean check) {
- this.rollupHIIoptimizable = check;
- }
-
- public boolean getRollupHIIOptimizable() {
- return this.rollupHIIoptimizable;
- }
-
public UserFuncExpression(OperatorPlan plan, FuncSpec funcSpec) {
super("UserFunc", plan);
mFuncSpec = funcSpec;
@@ -87,6 +67,7 @@ public class UserFuncExpression extends
}
}
+
public UserFuncExpression(OperatorPlan plan, FuncSpec funcSpec, List<LogicalExpression> args) {
this( plan, funcSpec );
Modified: pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java Fri May 22 00:06:16 2015
@@ -44,7 +44,6 @@ import org.apache.pig.newplan.logical.ru
import org.apache.pig.newplan.logical.rules.PredicatePushdownOptimizer;
import org.apache.pig.newplan.logical.rules.PushDownForEachFlatten;
import org.apache.pig.newplan.logical.rules.PushUpFilter;
-import org.apache.pig.newplan.logical.rules.RollupHIIOptimizer;
import org.apache.pig.newplan.logical.rules.SplitFilter;
import org.apache.pig.newplan.logical.rules.StreamTypeCastInserter;
import org.apache.pig.newplan.optimizer.PlanOptimizer;
@@ -57,7 +56,6 @@ public class LogicalPlanOptimizer extend
private boolean allRulesDisabled = false;
private SetMultimap<RulesReportKey, String> rulesReport = TreeMultimap.create();
private PigContext pc = null;
- private static final String MAPREDUCE_FW = "MAPREDUCE";
public LogicalPlanOptimizer(OperatorPlan p, int iterations, Set<String> turnOffRules) {
this(p, iterations, turnOffRules, null);
@@ -205,20 +203,6 @@ public class LogicalPlanOptimizer extend
if (!s.isEmpty())
ls.add(s);
- // RollupHIIOptimizer Set
- // This set of rules for rollup hii
- // If pig is not running in MR mode, this rule will be disabled
- if (pc!=null)
- if (pc.getExecType().toString().equals(MAPREDUCE_FW)) {
- s = new HashSet<Rule>();
- // Optimize RollupHII
- r = new RollupHIIOptimizer("RollupHIIOptimizer");
- checkAndAddRule(s, r);
- if (!s.isEmpty())
- ls.add(s);
- } else {
- LOG.info("Not MR mode. RollupHIIOptimizer is disabled");
- }
return ls;
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java Fri May 22 00:06:16 2015
@@ -70,60 +70,7 @@ public class LOCogroup extends LogicalRe
* static constant to refer to the option of selecting a group type
*/
public final static Integer OPTION_GROUPTYPE = 1;
-
- //the pivot value
- private int pivot = -1;
- //the index of the first field involves in ROLLUP
- private int rollupFieldIndex = 0;
- //the original index of the first field involves in ROLLUP in case it was moved to the end
- //(if we have the combination of cube and rollup)
- private int rollupOldFieldIndex = 0;
- //the size of total fields that involve in CUBE clause
- private int dimensionSize = 0;
-
- //number of algebraic function that used after rollup
- private int nAlgebraic = 0;
-
- public void setPivot(int pvt) {
- this.pivot = pvt;
- }
-
- public int getPivot() {
- return this.pivot;
- }
-
- public void setDimensionSize(int ds) {
- this.dimensionSize = ds;
- }
-
- public int getDimensionSize() {
- return this.dimensionSize;
- }
-
- public void setNumberAlgebraic(int na) {
- this.nAlgebraic = na;
- }
-
- public int getNumberAlgebraic() {
- return this.nAlgebraic;
- }
-
- public void setRollupOldFieldIndex(int rofi) {
- this.rollupOldFieldIndex = rofi;
- }
-
- public int getRollupOldFieldIndex() {
- return this.rollupOldFieldIndex;
- }
-
- public void setRollupFieldIndex(int rfi) {
- this.rollupFieldIndex = rfi;
- }
-
- public int getRollupFieldIndex() {
- return this.rollupFieldIndex;
- }
-
+
/**
* Constructor for use in defining rule patterns
* @param plan
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCube.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCube.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCube.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCube.java Fri May 22 00:06:16 2015
@@ -84,16 +84,6 @@ import org.apache.pig.newplan.logical.ex
public class LOCube extends LogicalRelationalOperator {
private MultiMap<Integer, LogicalExpressionPlan> mExpressionPlans;
private List<String> operations;
- //the pivot position
- private int pivot = -1;
-
- public void setPivot(int pvt) {
- this.pivot = pvt;
- }
-
- public int getPivot() {
- return this.pivot;
- }
public LOCube(LogicalPlan plan) {
super("LOCube", plan);
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java Fri May 22 00:06:16 2015
@@ -58,7 +58,6 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PONative;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORollupHIIForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
@@ -902,128 +901,6 @@ public class LogToPhyTranslationVisitor
translateSoftLinks(foreach);
}
- @Override
- public void visit(LORollupHIIForEach hforeach) throws FrontendException {
- String scope = DEFAULT_SCOPE;
-
- List<PhysicalPlan> innerPlans = new ArrayList<PhysicalPlan>();
-
- org.apache.pig.newplan.logical.relational.LogicalPlan inner = hforeach.getInnerPlan();
- LOGenerate gen = (LOGenerate) inner.getSinks().get(0);
-
- List<LogicalExpressionPlan> exps = gen.getOutputPlans();
- List<Operator> preds = inner.getPredecessors(gen);
-
- currentPlans.push(currentPlan);
-
- // we need to translate each predecessor of LOGenerate into a physical plan.
- // The physical plan should contain the expression plan for this predecessor plus
- // the subtree starting with this predecessor
- for (int i = 0; i < exps.size(); i++) {
- currentPlan = new PhysicalPlan();
- // translate the expression plan
- PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(exps.get(i));
- pushWalker(childWalker);
- childWalker.walk(new ExpToPhyTranslationVisitor(exps.get(i), childWalker, gen,
- currentPlan, logToPhyMap));
- popWalker();
-
- List<Operator> leaves = exps.get(i).getSinks();
- for (Operator l : leaves) {
- PhysicalOperator op = logToPhyMap.get(l);
- if (l instanceof ProjectExpression) {
- int input = ((ProjectExpression) l).getInputNum();
-
- // for each sink projection, get its input logical plan and translate it
- Operator pred = preds.get(input);
- childWalker = new SubtreeDependencyOrderWalker(inner, pred);
- pushWalker(childWalker);
- childWalker.walk(this);
- popWalker();
-
- // get the physical operator of the leaf of input logical plan
- PhysicalOperator leaf = logToPhyMap.get(pred);
-
- if (pred instanceof LOInnerLoad) {
- // if predecessor is only an LOInnerLoad, remove the project that
- // comes from LOInnerLoad and change the column of project that
- // comes from expression plan
- currentPlan.remove(leaf);
- logToPhyMap.remove(pred);
-
- POProject leafProj = (POProject) leaf;
- try {
- if (leafProj.isStar()) {
- ((POProject) op).setStar(true);
- } else if (leafProj.isProjectToEnd()) {
- ((POProject) op).setProjectToEnd(leafProj.getStartCol());
- } else {
- ((POProject) op).setColumn(leafProj.getColumn());
- }
-
- } catch (ExecException e) {
- throw new FrontendException(hforeach, "Cannot get column from " + leaf,
- 2230, e);
- }
-
- } else {
- currentPlan.connect(leaf, op);
- }
- }
- }
- innerPlans.add(currentPlan);
- }
-
- currentPlan = currentPlans.pop();
-
- // PhysicalOperator poGen = new POGenerate(new OperatorKey("",
- // r.nextLong()), inputs, toBeFlattened);
- boolean[] flatten = gen.getFlattenFlags();
- List<Boolean> flattenList = new ArrayList<Boolean>();
- for (boolean fl : flatten) {
- flattenList.add(fl);
- }
- // Create new PORollupHIIForEach for translation from Logical Plan to Physical Plan
- PORollupHIIForEach poHFE = new PORollupHIIForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
- hforeach.getRequestedParallelism(), innerPlans, flattenList);
-
- // if the pivot position is zero, set the pivot position for physical op is zero
- if(hforeach.getPivot() == 0)
- poHFE.setPivot(0);
- //else, decrease pivot position by one, because the position user specified and the
- //rollup field index is different by one
- else
- poHFE.setPivot(hforeach.getPivot() - 1);
- //get the start field index and size of rollup position in case the rollup does not stand at the front
- poHFE.setRollupFieldIndex(hforeach.getRollupFieldIndex());
- poHFE.setRollupOldFieldIndex(hforeach.getRollupOldFieldIndex());
- poHFE.setRollupSize(hforeach.getRollupSize());
- poHFE.setDimensionSize(hforeach.getDimensionSize());
-
- poHFE.addOriginalLocation(hforeach.getAlias(), hforeach.getLocation());
- poHFE.setResultType(DataType.BAG);
- logToPhyMap.put(hforeach, poHFE);
- currentPlan.add(poHFE);
-
- // generate cannot have multiple inputs
- List<Operator> op = hforeach.getPlan().getPredecessors(hforeach);
-
- // generate may not have any predecessors
- if (op == null)
- return;
-
- PhysicalOperator from = logToPhyMap.get(op.get(0));
- try {
- currentPlan.connect(from, poHFE);
- } catch (Exception e) {
- int errCode = 2015;
- String msg = "Invalid physical operators in the physical plan";
- throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
- }
-
- translateSoftLinks(hforeach);
- }
-
/**
* This function takes in a List of LogicalExpressionPlan and converts them to
* a list of PhysicalPlans
@@ -1134,19 +1011,6 @@ public class LogToPhyTranslationVisitor
case REGULAR:
POPackage poPackage = compileToLR_GR_PackTrio(cg, cg.getCustomPartitioner(), cg.getInner(), cg.getExpressionPlans());
poPackage.getPkgr().setPackageType(PackageType.GROUP);
- if(cg.getPivot()!=-1) {
- //Set the pivot value
- poPackage.setPivot(cg.getPivot());
- //Set the size of total fields that involve in CUBE clause
- poPackage.setDimensionSize(cg.getDimensionSize());
- //Set the index of the first field involves in ROLLUP
- poPackage.setRollupFieldIndex(cg.getRollupFieldIndex());
- //Set the original index of the first field involves in ROLLUP in case it was moved to the end
- //(if we have the combination of cube and rollup)
- poPackage.setRollupOldFieldIndex(cg.getRollupOldFieldIndex());
- //Set number of algebraic functions that used after rollup
- poPackage.setNumberAlgebraic(cg.getNumberAlgebraic());
- }
logToPhyMap.put(cg, poPackage);
break;
case MERGE:
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java Fri May 22 00:06:16 2015
@@ -263,7 +263,6 @@ public class LogicalPlan extends BaseOpe
disabledOptimizerRules.add("ColumnMapKeyPrune");
disabledOptimizerRules.add("AddForEach");
disabledOptimizerRules.add("GroupByConstParallelSetter");
- disabledOptimizerRules.add("RollupHIIOptimizer");
}
try {
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java Fri May 22 00:06:16 2015
@@ -57,9 +57,6 @@ public abstract class LogicalRelationalN
public void visit(LOForEach foreach) throws FrontendException {
}
- public void visit(LORollupHIIForEach horeach) throws FrontendException {
- }
-
public void visit(LOGenerate gen) throws FrontendException {
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java Fri May 22 00:06:16 2015
@@ -27,7 +27,6 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.UserFuncExpression;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOGenerate;
-import org.apache.pig.newplan.logical.relational.LORollupHIIForEach;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
public class OptimizerUtils {
@@ -42,16 +41,6 @@ public class OptimizerUtils {
}
/**
- * Find generate op from the rolluphiiforeach operator.
- * @param foreach the LORollupHIIForEach instance
- * @return LOGenerate instance
- */
- public static LOGenerate findGenerate(LORollupHIIForEach hfe) {
- LogicalPlan inner = hfe.getInnerPlan();
- return (LOGenerate) inner.getSinks().get(0);
- }
-
- /**
* Check if a given LOGenerate operator has any flatten fields.
* @param gen the given LOGenerate instance
* @return true if LOGenerate instance contains flatten fields, false otherwise
Modified: pig/trunk/src/org/apache/pig/parser/AliasMasker.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AliasMasker.g?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/AliasMasker.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AliasMasker.g Fri May 22 00:06:16 2015
@@ -247,10 +247,6 @@ cube_clause
: ^( CUBE cube_item )
;
-pivot_clause
- : ^( PIVOT INTEGER )
-;
-
cube_item
: rel ( cube_by_clause )
;
@@ -264,7 +260,7 @@ cube_or_rollup
;
cube_rollup_list
- : ^( CUBE cube_by_expr_list ) | ^( ROLLUP cube_by_expr_list pivot_clause? )
+ : ^( ( CUBE | ROLLUP ) cube_by_expr_list )
;
cube_by_expr_list
@@ -646,7 +642,6 @@ eid : rel_str_op
| FOREACH
| CUBE
| ROLLUP
- | PIVOT
| MATCHES
| ORDER
| RANK
Modified: pig/trunk/src/org/apache/pig/parser/AstPrinter.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AstPrinter.g?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/AstPrinter.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AstPrinter.g Fri May 22 00:06:16 2015
@@ -106,10 +106,6 @@ parallel_clause
: ^( PARALLEL INTEGER ) { sb.append(" ").append($PARALLEL.text).append(" ").append($INTEGER.text); }
;
-pivot_clause
- : ^( PIVOT INTEGER ) { sb.append(" ").append($PIVOT.text).append(" ").append($INTEGER.text); }
-;
-
alias
: IDENTIFIER { sb.append($IDENTIFIER.text); }
;
@@ -266,7 +262,7 @@ cube_or_rollup
;
cube_rollup_list
- : ^( CUBE { sb.append($CUBE.text).append("("); } cube_by_expr_list { sb.append(")"); } ) | ^( ROLLUP { sb.append($ROLLUP.text).append("("); } cube_by_expr_list { sb.append(")"); } )
+ : ^( ( CUBE { sb.append($CUBE.text).append("("); } | ROLLUP { sb.append($ROLLUP.text).append("("); } ) cube_by_expr_list { sb.append(")"); })
;
cube_by_expr_list
@@ -274,7 +270,7 @@ cube_by_expr_list
;
cube_by_expr
- : col_range | expr | STAR { sb.append($STAR.text); } { sb.append(" "); }
+ : col_range | expr | STAR { sb.append($STAR.text); }
;
group_clause
@@ -676,7 +672,6 @@ eid : rel_str_op
| FOREACH { sb.append($FOREACH.text); }
| CUBE { sb.append($CUBE.text); }
| ROLLUP { sb.append($ROLLUP.text); }
- | PIVOT { sb.append($PIVOT.text); }
| MATCHES { sb.append($MATCHES.text); }
| ORDER { sb.append($ORDER.text); }
| RANK { sb.append($RANK.text); }
Modified: pig/trunk/src/org/apache/pig/parser/AstValidator.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AstValidator.g?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/AstValidator.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AstValidator.g Fri May 22 00:06:16 2015
@@ -296,10 +296,6 @@ cube_clause
: ^( CUBE cube_item )
;
-pivot_clause
- : ^( PIVOT INTEGER )
-;
-
cube_item
: rel ( cube_by_clause )
;
@@ -313,7 +309,7 @@ cube_or_rollup
;
cube_rollup_list
- : ^( CUBE cube_by_expr_list ) | ^( ROLLUP cube_by_expr_list pivot_clause? )
+ : ^( ( CUBE | ROLLUP ) cube_by_expr_list )
;
cube_by_expr_list
@@ -667,7 +663,6 @@ eid : rel_str_op
| FOREACH
| CUBE
| ROLLUP
- | PIVOT
| MATCHES
| ORDER
| RANK
Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java Fri May 22 00:06:16 2015
@@ -451,30 +451,10 @@ public class LogicalPlanBuilder {
return new LOCube(plan);
}
- void setPivotRollupCubeOp(LOCube op, Integer pivot) throws ParserValidationException {
- if(pivot!=null)
- op.setPivot(pivot);
- }
-
String buildCubeOp(SourceLocation loc, LOCube op, String alias, String inputAlias,
List<String> operations, MultiMap<Integer, LogicalExpressionPlan> expressionPlans)
throws ParserValidationException {
- // check value of pivot if it is valid or not, if not pivot position
- // is specified, the pivot at middle position will be chosen
- try {
- if(op.getPivot()!=-1) {
- if (op.getPivot() < 0 || op.getPivot() >= expressionPlans.get(0).size()) {
- FrontendException fe = new FrontendException("PIVOT is out of bound");
- throw fe;
- }
- }
- else
- op.setPivot((int)(Math.round(expressionPlans.get(0).size()/2.0)));
- } catch (FrontendException e) {
- throw new ParserValidationException(intStream, loc, e);
- }
-
// check if continuously occurring cube operations be combined
combineCubeOperations((ArrayList<String>) operations, expressionPlans);
@@ -733,7 +713,6 @@ public class LogicalPlanBuilder {
// build group by operator
try {
- groupby.setPivot(op.getPivot());
return buildGroupOp(loc, (LOCogroup) groupby, op.getAlias(), inpAliases, exprPlansCopy,
GROUPTYPE.REGULAR, innerFlags, null);
} catch (ParserValidationException pve) {
Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g Fri May 22 00:06:16 2015
@@ -493,20 +493,12 @@ func_args returns[List<String> args]
// It also outputs the order of operations i.e in this case CUBE operation followed by ROLLUP operation
// These inputs are passed to buildCubeOp methods which then builds the logical plan for CUBE operator.
// If user specifies STAR or RANGE expression for dimensions then it will be expanded inside buildCubeOp.
-pivot_clause returns[int pivot]
- : ^( PIVOT INTEGER )
- {
- $pivot = Integer.parseInt( $INTEGER.text );
- }
-;
-
cube_clause returns[String alias]
scope {
LOCube cubeOp;
MultiMap<Integer, LogicalExpressionPlan> cubePlans;
List<String> operations;
int inputIndex;
- int pivot;
}
scope GScope;
@init {
@@ -556,7 +548,7 @@ cube_rollup_list returns[String operatio
@init {
$plans = new ArrayList<LogicalExpressionPlan>();
}
- : ^( CUBE { $operation = "CUBE"; } cube_by_expr_list { $plans = $cube_by_expr_list.plans; } ) | ^( ROLLUP { $operation = "ROLLUP"; } cube_by_expr_list { $plans = $cube_by_expr_list.plans; } pivot_clause? { if ($pivot_clause.tree!=null) builder.setPivotRollupCubeOp($cube_clause::cubeOp, $pivot_clause.pivot); } )
+ : ^( ( CUBE { $operation = "CUBE"; } | ROLLUP { $operation = "ROLLUP"; } ) cube_by_expr_list { $plans = $cube_by_expr_list.plans; } )
;
cube_by_expr_list returns[List<LogicalExpressionPlan> plans]
@@ -1949,7 +1941,6 @@ eid returns[String id] : rel_str_op { $i
| COGROUP { $id = $COGROUP.text; }
| CUBE { $id = $CUBE.text; }
| ROLLUP { $id = $ROLLUP.text; }
- | PIVOT { $id = $PIVOT.text; }
| JOIN { $id = $JOIN.text; }
| CROSS { $id = $CROSS.text; }
| UNION { $id = $UNION.text; }
Modified: pig/trunk/src/org/apache/pig/parser/QueryLexer.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryLexer.g?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryLexer.g (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryLexer.g Fri May 22 00:06:16 2015
@@ -153,9 +153,6 @@ ONSCHEMA : 'ONSCHEMA'
PARALLEL : 'PARALLEL'
;
-PIVOT : 'PIVOT'
-;
-
PARTITION : 'PARTITION'
;
Modified: pig/trunk/src/org/apache/pig/parser/QueryParser.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParser.g?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParser.g (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParser.g Fri May 22 00:06:16 2015
@@ -595,10 +595,7 @@ union_clause : UNION^ ONSCHEMA? rel_list
cube_clause : CUBE rel BY cube_rollup_list ( COMMA cube_rollup_list )* -> ^( CUBE rel ^( BY cube_rollup_list+ ) )
;
-cube_rollup_list : ( CUBE^ LEFT_PAREN! real_arg ( COMMA! real_arg )* RIGHT_PAREN! ) | ( ROLLUP^ LEFT_PAREN! real_arg ( COMMA! real_arg )* RIGHT_PAREN! pivot_clause? )
-;
-
-pivot_clause : PIVOT^ INTEGER
+cube_rollup_list : ( CUBE | ROLLUP )^ LEFT_PAREN! real_arg ( COMMA! real_arg )* RIGHT_PAREN!
;
flatten_clause : FLATTEN^ LEFT_PAREN! expr RIGHT_PAREN!
Modified: pig/trunk/test/org/apache/pig/test/TestCubeOperator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestCubeOperator.java?rev=1680969&r1=1680968&r2=1680969&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestCubeOperator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestCubeOperator.java Fri May 22 00:06:16 2015
@@ -143,34 +143,6 @@ public class TestCubeOperator {
}
@Test
- public void testRollupHIIBasic() throws IOException {
- // basic correctness test
- String query = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
- + "b = cube a by rollup(x,y) pivot 1;"
- + "c = foreach b generate flatten(group) as (type,location), COUNT_STAR(cube) as count, SUM(cube.z) as total;"
- + "store c into 'output' using mock.Storage();";
- Util.registerMultiLineQuery(pigServer, query);
-
- Set<Tuple> expected = ImmutableSet.of(
- tf.newTuple(Lists.newArrayList("cat", "miami", (long) 1, (long) 18)),
- tf.newTuple(Lists.newArrayList("cat", "naples", (long) 1, (long) 9)),
- tf.newTuple(Lists.newArrayList("cat", null, (long) 2, (long) 27)),
- tf.newTuple(Lists.newArrayList("dog", "miami", (long) 1, (long) 12)),
- tf.newTuple(Lists.newArrayList("dog", "tampa", (long) 1, (long) 14)),
- tf.newTuple(Lists.newArrayList("dog", "naples", (long) 1, (long) 5)),
- tf.newTuple(Lists.newArrayList("dog", null, (long) 3, (long) 31)),
- tf.newTuple(Lists.newArrayList("turtle", "tampa", (long) 1, (long) 4)),
- tf.newTuple(Lists.newArrayList("turtle", "naples", (long) 1, (long) 1)),
- tf.newTuple(Lists.newArrayList("turtle", null, (long) 2, (long) 5)),
- tf.newTuple(Lists.newArrayList(null, null, (long) 7, (long) 63)));
-
- List<Tuple> out = data.get("output");
- for (Tuple tup : out) {
- assertTrue(expected + " contains " + tup, expected.contains(tup));
- }
- }
-
- @Test
public void testCubeAndRollup() throws IOException {
// basic correctness test
String query = "a = load 'input2' USING mock.Storage() as (v:chararray,w:chararray,x:chararray,y:chararray,z:long);"
@@ -203,38 +175,6 @@ public class TestCubeOperator {
}
@Test
- public void testCubeAndRollupHII() throws IOException {
- // basic correctness test
- String query = "a = load 'input2' USING mock.Storage() as (v:chararray,w:chararray,x:chararray,y:chararray,z:long);"
- + "b = cube a by cube(v,w), rollup(x,y) pivot 1;"
- + "c = foreach b generate flatten(group) as (type,location,color,category), COUNT_STAR(cube) as count, SUM(cube.z) as total;"
- + "store c into 'output' using mock.Storage();";
- Util.registerMultiLineQuery(pigServer, query);
-
- Set<Tuple> expected = ImmutableSet
- .of(tf.newTuple(Lists.newArrayList("dog", "miami", "white", "pet", (long) 1,
- (long) 5)), tf.newTuple(Lists.newArrayList("dog", null, "white", "pet",
- (long) 1, (long) 5)), tf.newTuple(Lists.newArrayList(null, "miami",
- "white", "pet", (long) 1, (long) 5)), tf.newTuple(Lists.newArrayList(null,
- null, "white", "pet", (long) 1, (long) 5)), tf.newTuple(Lists.newArrayList(
- "dog", "miami", "white", null, (long) 1, (long) 5)), tf.newTuple(Lists
- .newArrayList("dog", null, "white", null, (long) 1, (long) 5)), tf
- .newTuple(Lists.newArrayList(null, "miami", "white", null, (long) 1,
- (long) 5)), tf.newTuple(Lists.newArrayList(null, null, "white",
- null, (long) 1, (long) 5)), tf.newTuple(Lists.newArrayList("dog", "miami",
- null, null, (long) 1, (long) 5)), tf.newTuple(Lists.newArrayList("dog",
- null, null, null, (long) 1, (long) 5)), tf.newTuple(Lists.newArrayList(
- null, "miami", null, null, (long) 1, (long) 5)), tf.newTuple(Lists
- .newArrayList(null, null, null, null, (long) 1, (long) 5)));
-
- List<Tuple> out = data.get("output");
- for (Tuple tup : out) {
- assertTrue(expected + " contains " + tup, expected.contains(tup));
- }
-
- }
-
- @Test
public void testCubeMultipleIAliases() throws IOException {
// test for input alias to cube being assigned multiple times
String query = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
@@ -635,36 +575,6 @@ public class TestCubeOperator {
}
@Test
- public void testRollupHIIAfterCogroup() throws IOException {
- // test for cubing on co-grouped relation
- String query = "a = load 'input1' USING mock.Storage() as (a1:chararray,b1,c1,d1); "
- + "b = load 'input' USING mock.Storage() as (a2,b2,c2:long,d2:chararray);"
- + "c = cogroup a by a1, b by d2;"
- + "d = foreach c generate flatten(a), flatten(b);"
- + "e = cube d by rollup(a2,b2) pivot 1;"
- + "f = foreach e generate flatten(group), COUNT(cube) as count, SUM(cube.c2) as total;"
- + "store f into 'output' using mock.Storage();";
-
- Util.registerMultiLineQuery(pigServer, query);
-
- Set<Tuple> expected = ImmutableSet.of(
- tf.newTuple(Lists.newArrayList("cat", "miami", (long) 1, (long) 18)),
- tf.newTuple(Lists.newArrayList("cat", null, (long) 1, (long) 18)),
- tf.newTuple(Lists.newArrayList("dog", "miami", (long) 1, (long) 12)),
- tf.newTuple(Lists.newArrayList("dog", "tampa", (long) 1, (long) 14)),
- tf.newTuple(Lists.newArrayList("dog", null, (long) 2, (long) 26)),
- tf.newTuple(Lists.newArrayList("turtle", "tampa", (long) 1, (long) 4)),
- tf.newTuple(Lists.newArrayList("turtle", "naples", (long) 1, (long) 1)),
- tf.newTuple(Lists.newArrayList("turtle", null, (long) 2, (long) 5)),
- tf.newTuple(Lists.newArrayList(null, null, (long) 5, (long) 49)));
-
- List<Tuple> out = data.get("output");
- for (Tuple tup : out) {
- assertTrue(expected + " contains " + tup, expected.contains(tup));
- }
- }
-
- @Test
public void testExplainCube() throws IOException {
// test for explain
String query = "a = load 'input' USING mock.Storage() as (a1:chararray,b1:chararray,c1:long); "
@@ -685,19 +595,6 @@ public class TestCubeOperator {
Util.registerMultiLineQuery(pigServer, query);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintStream ps = new PrintStream(baos);
- pigServer.explain("b", ps);
- assertTrue(baos.toString().contains("RollupDimensions"));
- }
-
- @Test
- public void testExplainRollupHII() throws IOException {
- // test for explain
- String query = "a = load 'input' USING mock.Storage() as (a1:chararray,b1:chararray,c1:long); "
- + "b = cube a by rollup(a1,b1) pivot 1;";
-
- Util.registerMultiLineQuery(pigServer, query);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
pigServer.explain("b", ps);
assertTrue(baos.toString().contains("RollupDimensions"));