You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2015/01/27 03:27:49 UTC
svn commit: r1654955 [3/10] - in /pig/branches/spark: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/xml/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/
contrib/piggybank/java/src/test/java/org/apach...
Modified: pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LOCogroup.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LOCogroup.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LOCogroup.java (original)
+++ pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LOCogroup.java Tue Jan 27 02:27:45 2015
@@ -37,13 +37,13 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
public class LOCogroup extends LogicalRelationalOperator {
-
+
// List of booleans specifying if any of the cogroups is inner
private boolean[] mIsInner;
-
+
// List of expressionPlans according to input
private MultiMap<Integer,LogicalExpressionPlan> mExpressionPlans;
-
+
/**
* Enum for the type of group
*/
@@ -52,39 +52,92 @@ public class LOCogroup extends LogicalRe
COLLECTED, // Collected group
MERGE // Map-side CoGroup on sorted data
};
-
+
private GROUPTYPE mGroupType;
-
- private LogicalFieldSchema groupKeyUidOnlySchema;
-
+
+ private LogicalFieldSchema groupKeyUidOnlySchema;
+
/*
* This is a map storing Uids which have been generated for an input
* This map is required to make the uids persistant between calls of
* resetSchema and getSchema
*/
private Map<Integer,Long> generatedInputUids = new HashMap<Integer,Long>();
-
+
final static String GROUP_COL_NAME = "group";
-
- /**
+
+ /**
* static constant to refer to the option of selecting a group type
*/
public final static Integer OPTION_GROUPTYPE = 1;
-
+
+ //the pivot value
+ private int pivot = -1;
+ //the index of the first field involves in ROLLUP
+ private int rollupFieldIndex = 0;
+ //the original index of the first field involves in ROLLUP in case it was moved to the end
+ //(if we have the combination of cube and rollup)
+ private int rollupOldFieldIndex = 0;
+ //the size of total fields that involve in CUBE clause
+ private int dimensionSize = 0;
+
+ //number of algebraic function that used after rollup
+ private int nAlgebraic = 0;
+
+ public void setPivot(int pvt) {
+ this.pivot = pvt;
+ }
+
+ public int getPivot() {
+ return this.pivot;
+ }
+
+ public void setDimensionSize(int ds) {
+ this.dimensionSize = ds;
+ }
+
+ public int getDimensionSize() {
+ return this.dimensionSize;
+ }
+
+ public void setNumberAlgebraic(int na) {
+ this.nAlgebraic = na;
+ }
+
+ public int getNumberAlgebraic() {
+ return this.nAlgebraic;
+ }
+
+ public void setRollupOldFieldIndex(int rofi) {
+ this.rollupOldFieldIndex = rofi;
+ }
+
+ public int getRollupOldFieldIndex() {
+ return this.rollupOldFieldIndex;
+ }
+
+ public void setRollupFieldIndex(int rfi) {
+ this.rollupFieldIndex = rfi;
+ }
+
+ public int getRollupFieldIndex() {
+ return this.rollupFieldIndex;
+ }
+
/**
* Constructor for use in defining rule patterns
* @param plan
*/
public LOCogroup(LogicalPlan plan) {
- super("LOCogroup", plan);
+ super("LOCogroup", plan);
}
-
- public LOCogroup(OperatorPlan plan, MultiMap<Integer,LogicalExpressionPlan>
+
+ public LOCogroup(OperatorPlan plan, MultiMap<Integer,LogicalExpressionPlan>
expressionPlans, boolean[] isInner ) {
this( plan, expressionPlans, GROUPTYPE.REGULAR, isInner );
}
- public LOCogroup(OperatorPlan plan, MultiMap<Integer,LogicalExpressionPlan>
+ public LOCogroup(OperatorPlan plan, MultiMap<Integer,LogicalExpressionPlan>
expressionPlans, GROUPTYPE groupType, boolean[] isInner) {
super("LOCogroup", plan);
this.mExpressionPlans = expressionPlans;
@@ -93,7 +146,7 @@ public class LOCogroup extends LogicalRe
}
this.mGroupType = groupType;
}
-
+
/**
* Given an expression plan this function returns a LogicalFieldSchema
* that can be generated using this expression plan
@@ -120,7 +173,7 @@ public class LOCogroup extends LogicalRe
if (inputs == null) {
throw new FrontendException(this, "Cannot get predecessor for " + this, 2233);
}
-
+
List<LogicalFieldSchema> fieldSchemaList = new ArrayList<LogicalFieldSchema>();
// See if we have more than one expression plans, if so the
@@ -139,7 +192,7 @@ public class LOCogroup extends LogicalRe
LogicalSchema keySchema = new LogicalSchema();
// We sort here to maintain the correct order of inputs
for( Integer key : mExpressionPlans.keySet()) {
- Collection<LogicalExpressionPlan> plans =
+ Collection<LogicalExpressionPlan> plans =
mExpressionPlans.get(key);
for( LogicalExpressionPlan plan : plans ) {
@@ -175,14 +228,14 @@ public class LOCogroup extends LogicalRe
break;
}
break;
- }
+ }
}
if(mExpressionPlans.size() > 1){
//reset the uid, because the group column is associated with more
// than one input
groupKeySchema.resetUid();
}
-
+
if (groupKeySchema==null) {
throw new FrontendException(this, "Cannot get group key schema for " + this, 2234);
}
@@ -194,8 +247,8 @@ public class LOCogroup extends LogicalRe
int counter = 0;
for (Operator op : inputs) {
LogicalSchema inputSchema = ((LogicalRelationalOperator)op).getSchema();
-
- // Check if we already have calculated Uid for this bag for given
+
+ // Check if we already have calculated Uid for this bag for given
// input operator
long bagUid;
if (generatedInputUids.get(counter)!=null)
@@ -204,15 +257,15 @@ public class LOCogroup extends LogicalRe
bagUid = LogicalExpression.getNextUid();
generatedInputUids.put( counter, bagUid );
}
-
+
LogicalFieldSchema newTupleFieldSchema = new LogicalFieldSchema(
null, inputSchema, DataType.TUPLE, LogicalExpression.getNextUid());
-
+
LogicalSchema bagSchema = new LogicalSchema();
bagSchema.addField(newTupleFieldSchema);
-
+
LogicalFieldSchema newBagFieldSchema = new LogicalFieldSchema(
- ((LogicalRelationalOperator)op).getAlias(), bagSchema,
+ ((LogicalRelationalOperator)op).getAlias(), bagSchema,
DataType.BAG, bagUid);
fieldSchemaList.add( newBagFieldSchema );
@@ -222,7 +275,7 @@ public class LOCogroup extends LogicalRe
schema = new LogicalSchema();
for(LogicalFieldSchema fieldSchema: fieldSchemaList) {
schema.addField(fieldSchema);
- }
+ }
return schema;
}
@@ -239,32 +292,32 @@ public class LOCogroup extends LogicalRe
public boolean isEqual(Operator other) throws FrontendException {
if (other != null && other instanceof LOCogroup) {
LOCogroup oc = (LOCogroup)other;
- if( mGroupType == oc.mGroupType &&
- mIsInner.length == oc.mIsInner.length
+ if( mGroupType == oc.mGroupType &&
+ mIsInner.length == oc.mIsInner.length
&& mExpressionPlans.size() == oc.mExpressionPlans.size() ) {
for( int i = 0; i < mIsInner.length; i++ ) {
if( mIsInner[i] != oc.mIsInner[i] ) {
return false;
}
}
- for( Integer key : mExpressionPlans.keySet() ) {
+ for( Integer key : mExpressionPlans.keySet() ) {
if( ! oc.mExpressionPlans.containsKey(key) ) {
return false;
}
- Collection<LogicalExpressionPlan> exp1 =
+ Collection<LogicalExpressionPlan> exp1 =
mExpressionPlans.get(key);
- Collection<LogicalExpressionPlan> exp2 =
+ Collection<LogicalExpressionPlan> exp2 =
oc.mExpressionPlans.get(key);
- if(! ( exp1 instanceof ArrayList<?>
+ if(! ( exp1 instanceof ArrayList<?>
|| exp2 instanceof ArrayList<?> ) ) {
throw new FrontendException( "Expected an ArrayList " +
"of Expression Plans", 2235 );
}
- ArrayList<LogicalExpressionPlan> expList1 =
+ ArrayList<LogicalExpressionPlan> expList1 =
(ArrayList<LogicalExpressionPlan>) exp1;
- ArrayList<LogicalExpressionPlan> expList2 =
+ ArrayList<LogicalExpressionPlan> expList2 =
(ArrayList<LogicalExpressionPlan>) exp2;
for (int i = 0; i < expList1.size(); i++) {
@@ -282,37 +335,37 @@ public class LOCogroup extends LogicalRe
public GROUPTYPE getGroupType() {
return mGroupType;
}
-
+
public void resetGroupType() {
mGroupType = GROUPTYPE.REGULAR;
}
-
+
/**
- * Returns an Unmodifiable Map of Input Number to Uid
+ * Returns an Unmodifiable Map of Input Number to Uid
* @return Unmodifiable Map<Integer,Long>
*/
public Map<Integer,Long> getGeneratedInputUids() {
return Collections.unmodifiableMap( generatedInputUids );
}
-
+
public MultiMap<Integer,LogicalExpressionPlan> getExpressionPlans() {
return mExpressionPlans;
}
-
+
public void setExpressionPlans(MultiMap<Integer,LogicalExpressionPlan> plans) {
this.mExpressionPlans = plans;
}
-
+
public void setGroupType(GROUPTYPE gt) {
mGroupType = gt;
}
-
+
public void setInnerFlags(boolean[] flags) {
if( flags != null ) {
mIsInner = Arrays.copyOf( flags, flags.length );
}
}
-
+
public boolean[] getInner() {
return mIsInner;
}
@@ -322,7 +375,7 @@ public class LOCogroup extends LogicalRe
groupKeyUidOnlySchema = null;
generatedInputUids = new HashMap<Integer,Long>();
}
-
+
public List<Operator> getInputs(LogicalPlan plan) {
return plan.getPredecessors(this);
}
Modified: pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LOCube.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LOCube.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LOCube.java (original)
+++ pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LOCube.java Tue Jan 27 02:27:45 2015
@@ -84,6 +84,16 @@ import org.apache.pig.newplan.logical.ex
public class LOCube extends LogicalRelationalOperator {
private MultiMap<Integer, LogicalExpressionPlan> mExpressionPlans;
private List<String> operations;
+ //the pivot position
+ private int pivot = -1;
+
+ public void setPivot(int pvt) {
+ this.pivot = pvt;
+ }
+
+ public int getPivot() {
+ return this.pivot;
+ }
public LOCube(LogicalPlan plan) {
super("LOCube", plan);
Modified: pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java (original)
+++ pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java Tue Jan 27 02:27:45 2015
@@ -58,6 +58,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PONative;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORollupHIIForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
@@ -900,6 +901,128 @@ public class LogToPhyTranslationVisitor
translateSoftLinks(foreach);
}
+ @Override
+ public void visit(LORollupHIIForEach hforeach) throws FrontendException {
+ String scope = DEFAULT_SCOPE;
+
+ List<PhysicalPlan> innerPlans = new ArrayList<PhysicalPlan>();
+
+ org.apache.pig.newplan.logical.relational.LogicalPlan inner = hforeach.getInnerPlan();
+ LOGenerate gen = (LOGenerate) inner.getSinks().get(0);
+
+ List<LogicalExpressionPlan> exps = gen.getOutputPlans();
+ List<Operator> preds = inner.getPredecessors(gen);
+
+ currentPlans.push(currentPlan);
+
+ // we need to translate each predecessor of LOGenerate into a physical plan.
+ // The physical plan should contain the expression plan for this predecessor plus
+ // the subtree starting with this predecessor
+ for (int i = 0; i < exps.size(); i++) {
+ currentPlan = new PhysicalPlan();
+ // translate the expression plan
+ PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(exps.get(i));
+ pushWalker(childWalker);
+ childWalker.walk(new ExpToPhyTranslationVisitor(exps.get(i), childWalker, gen,
+ currentPlan, logToPhyMap));
+ popWalker();
+
+ List<Operator> leaves = exps.get(i).getSinks();
+ for (Operator l : leaves) {
+ PhysicalOperator op = logToPhyMap.get(l);
+ if (l instanceof ProjectExpression) {
+ int input = ((ProjectExpression) l).getInputNum();
+
+ // for each sink projection, get its input logical plan and translate it
+ Operator pred = preds.get(input);
+ childWalker = new SubtreeDependencyOrderWalker(inner, pred);
+ pushWalker(childWalker);
+ childWalker.walk(this);
+ popWalker();
+
+ // get the physical operator of the leaf of input logical plan
+ PhysicalOperator leaf = logToPhyMap.get(pred);
+
+ if (pred instanceof LOInnerLoad) {
+ // if predecessor is only an LOInnerLoad, remove the project that
+ // comes from LOInnerLoad and change the column of project that
+ // comes from expression plan
+ currentPlan.remove(leaf);
+ logToPhyMap.remove(pred);
+
+ POProject leafProj = (POProject) leaf;
+ try {
+ if (leafProj.isStar()) {
+ ((POProject) op).setStar(true);
+ } else if (leafProj.isProjectToEnd()) {
+ ((POProject) op).setProjectToEnd(leafProj.getStartCol());
+ } else {
+ ((POProject) op).setColumn(leafProj.getColumn());
+ }
+
+ } catch (ExecException e) {
+ throw new FrontendException(hforeach, "Cannot get column from " + leaf,
+ 2230, e);
+ }
+
+ } else {
+ currentPlan.connect(leaf, op);
+ }
+ }
+ }
+ innerPlans.add(currentPlan);
+ }
+
+ currentPlan = currentPlans.pop();
+
+ // PhysicalOperator poGen = new POGenerate(new OperatorKey("",
+ // r.nextLong()), inputs, toBeFlattened);
+ boolean[] flatten = gen.getFlattenFlags();
+ List<Boolean> flattenList = new ArrayList<Boolean>();
+ for (boolean fl : flatten) {
+ flattenList.add(fl);
+ }
+ // Create new PORollupHIIForEach for translation from Logical Plan to Physical Plan
+ PORollupHIIForEach poHFE = new PORollupHIIForEach(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
+ hforeach.getRequestedParallelism(), innerPlans, flattenList);
+
+ // if the pivot position is zero, set the pivot position for physical op is zero
+ if(hforeach.getPivot() == 0)
+ poHFE.setPivot(0);
+ //else, decrease pivot position by one, because the position user specified and the
+ //rollup field index is different by one
+ else
+ poHFE.setPivot(hforeach.getPivot() - 1);
+ //get the start field index and size of rollup position in case the rollup does not stand at the front
+ poHFE.setRollupFieldIndex(hforeach.getRollupFieldIndex());
+ poHFE.setRollupOldFieldIndex(hforeach.getRollupOldFieldIndex());
+ poHFE.setRollupSize(hforeach.getRollupSize());
+ poHFE.setDimensionSize(hforeach.getDimensionSize());
+
+ poHFE.addOriginalLocation(hforeach.getAlias(), hforeach.getLocation());
+ poHFE.setResultType(DataType.BAG);
+ logToPhyMap.put(hforeach, poHFE);
+ currentPlan.add(poHFE);
+
+ // generate cannot have multiple inputs
+ List<Operator> op = hforeach.getPlan().getPredecessors(hforeach);
+
+ // generate may not have any predecessors
+ if (op == null)
+ return;
+
+ PhysicalOperator from = logToPhyMap.get(op.get(0));
+ try {
+ currentPlan.connect(from, poHFE);
+ } catch (Exception e) {
+ int errCode = 2015;
+ String msg = "Invalid physical operators in the physical plan";
+ throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
+ }
+
+ translateSoftLinks(hforeach);
+ }
+
/**
* This function takes in a List of LogicalExpressionPlan and converts them to
* a list of PhysicalPlans
@@ -1010,6 +1133,19 @@ public class LogToPhyTranslationVisitor
case REGULAR:
POPackage poPackage = compileToLR_GR_PackTrio(cg, cg.getCustomPartitioner(), cg.getInner(), cg.getExpressionPlans());
poPackage.getPkgr().setPackageType(PackageType.GROUP);
+ if(cg.getPivot()!=-1) {
+ //Set the pivot value
+ poPackage.setPivot(cg.getPivot());
+ //Set the size of total fields that involve in CUBE clause
+ poPackage.setDimensionSize(cg.getDimensionSize());
+ //Set the index of the first field involves in ROLLUP
+ poPackage.setRollupFieldIndex(cg.getRollupFieldIndex());
+ //Set the original index of the first field involves in ROLLUP in case it was moved to the end
+ //(if we have the combination of cube and rollup)
+ poPackage.setRollupOldFieldIndex(cg.getRollupOldFieldIndex());
+ //Set number of algebraic functions that used after rollup
+ poPackage.setNumberAlgebraic(cg.getNumberAlgebraic());
+ }
logToPhyMap.put(cg, poPackage);
break;
case MERGE:
Modified: pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java (original)
+++ pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java Tue Jan 27 02:27:45 2015
@@ -263,6 +263,7 @@ public class LogicalPlan extends BaseOpe
disabledOptimizerRules.add("ColumnMapKeyPrune");
disabledOptimizerRules.add("AddForEach");
disabledOptimizerRules.add("GroupByConstParallelSetter");
+ disabledOptimizerRules.add("RollupHIIOptimizer");
}
try {
Modified: pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java (original)
+++ pig/branches/spark/src/org/apache/pig/newplan/logical/relational/LogicalRelationalNodesVisitor.java Tue Jan 27 02:27:45 2015
@@ -57,6 +57,9 @@ public abstract class LogicalRelationalN
public void visit(LOForEach foreach) throws FrontendException {
}
+ public void visit(LORollupHIIForEach horeach) throws FrontendException {
+ }
+
public void visit(LOGenerate gen) throws FrontendException {
}
Modified: pig/branches/spark/src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java (original)
+++ pig/branches/spark/src/org/apache/pig/newplan/logical/rules/OptimizerUtils.java Tue Jan 27 02:27:45 2015
@@ -27,6 +27,7 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.UserFuncExpression;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOGenerate;
+import org.apache.pig.newplan.logical.relational.LORollupHIIForEach;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
public class OptimizerUtils {
@@ -41,6 +42,16 @@ public class OptimizerUtils {
}
/**
+ * Find generate op from the rolluphiiforeach operator.
+ * @param foreach the LORollupHIIForEach instance
+ * @return LOGenerate instance
+ */
+ public static LOGenerate findGenerate(LORollupHIIForEach hfe) {
+ LogicalPlan inner = hfe.getInnerPlan();
+ return (LOGenerate) inner.getSinks().get(0);
+ }
+
+ /**
* Check if a given LOGenerate operator has any flatten fields.
* @param gen the given LOGenerate instance
* @return true if LOGenerate instance contains flatten fields, false otherwise
Modified: pig/branches/spark/src/org/apache/pig/parser/AliasMasker.g
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/parser/AliasMasker.g?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/parser/AliasMasker.g (original)
+++ pig/branches/spark/src/org/apache/pig/parser/AliasMasker.g Tue Jan 27 02:27:45 2015
@@ -247,6 +247,10 @@ cube_clause
: ^( CUBE cube_item )
;
+pivot_clause
+ : ^( PIVOT INTEGER )
+;
+
cube_item
: rel ( cube_by_clause )
;
@@ -260,7 +264,7 @@ cube_or_rollup
;
cube_rollup_list
- : ^( ( CUBE | ROLLUP ) cube_by_expr_list )
+ : ^( CUBE cube_by_expr_list ) | ^( ROLLUP cube_by_expr_list pivot_clause? )
;
cube_by_expr_list
@@ -642,6 +646,7 @@ eid : rel_str_op
| FOREACH
| CUBE
| ROLLUP
+ | PIVOT
| MATCHES
| ORDER
| RANK
Modified: pig/branches/spark/src/org/apache/pig/parser/AstPrinter.g
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/parser/AstPrinter.g?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/parser/AstPrinter.g (original)
+++ pig/branches/spark/src/org/apache/pig/parser/AstPrinter.g Tue Jan 27 02:27:45 2015
@@ -106,6 +106,10 @@ parallel_clause
: ^( PARALLEL INTEGER ) { sb.append(" ").append($PARALLEL.text).append(" ").append($INTEGER.text); }
;
+pivot_clause
+ : ^( PIVOT INTEGER ) { sb.append(" ").append($PIVOT.text).append(" ").append($INTEGER.text); }
+;
+
alias
: IDENTIFIER { sb.append($IDENTIFIER.text); }
;
@@ -262,7 +266,7 @@ cube_or_rollup
;
cube_rollup_list
- : ^( ( CUBE { sb.append($CUBE.text).append("("); } | ROLLUP { sb.append($ROLLUP.text).append("("); } ) cube_by_expr_list { sb.append(")"); })
+ : ^( CUBE { sb.append($CUBE.text).append("("); } cube_by_expr_list { sb.append(")"); } ) | ^( ROLLUP { sb.append($ROLLUP.text).append("("); } cube_by_expr_list { sb.append(")"); } )
;
cube_by_expr_list
@@ -270,7 +274,7 @@ cube_by_expr_list
;
cube_by_expr
- : col_range | expr | STAR { sb.append($STAR.text); }
+ : col_range | expr | STAR { sb.append($STAR.text); } { sb.append(" "); }
;
group_clause
@@ -672,6 +676,7 @@ eid : rel_str_op
| FOREACH { sb.append($FOREACH.text); }
| CUBE { sb.append($CUBE.text); }
| ROLLUP { sb.append($ROLLUP.text); }
+ | PIVOT { sb.append($PIVOT.text); }
| MATCHES { sb.append($MATCHES.text); }
| ORDER { sb.append($ORDER.text); }
| RANK { sb.append($RANK.text); }
Modified: pig/branches/spark/src/org/apache/pig/parser/AstValidator.g
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/parser/AstValidator.g?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/parser/AstValidator.g (original)
+++ pig/branches/spark/src/org/apache/pig/parser/AstValidator.g Tue Jan 27 02:27:45 2015
@@ -296,6 +296,10 @@ cube_clause
: ^( CUBE cube_item )
;
+pivot_clause
+ : ^( PIVOT INTEGER )
+;
+
cube_item
: rel ( cube_by_clause )
;
@@ -309,7 +313,7 @@ cube_or_rollup
;
cube_rollup_list
- : ^( ( CUBE | ROLLUP ) cube_by_expr_list )
+ : ^( CUBE cube_by_expr_list ) | ^( ROLLUP cube_by_expr_list pivot_clause? )
;
cube_by_expr_list
@@ -663,6 +667,7 @@ eid : rel_str_op
| FOREACH
| CUBE
| ROLLUP
+ | PIVOT
| MATCHES
| ORDER
| RANK
Modified: pig/branches/spark/src/org/apache/pig/parser/DryRunGruntParser.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/parser/DryRunGruntParser.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/parser/DryRunGruntParser.java (original)
+++ pig/branches/spark/src/org/apache/pig/parser/DryRunGruntParser.java Tue Jan 27 02:27:45 2015
@@ -379,4 +379,11 @@ public class DryRunGruntParser extends P
protected void printClear() {
}
+ @Override
+ protected void processDefault(String key, String value) throws IOException {
+ }
+
+ @Override
+ protected void processDeclare(String key, String value) throws IOException {
+ }
}
Modified: pig/branches/spark/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/branches/spark/src/org/apache/pig/parser/LogicalPlanBuilder.java Tue Jan 27 02:27:45 2015
@@ -451,10 +451,30 @@ public class LogicalPlanBuilder {
return new LOCube(plan);
}
+ void setPivotRollupCubeOp(LOCube op, Integer pivot) throws ParserValidationException {
+ if(pivot!=null)
+ op.setPivot(pivot);
+ }
+
String buildCubeOp(SourceLocation loc, LOCube op, String alias, String inputAlias,
List<String> operations, MultiMap<Integer, LogicalExpressionPlan> expressionPlans)
throws ParserValidationException {
+ // check value of pivot if it is valid or not, if not pivot position
+ // is specified, the pivot at middle position will be chosen
+ try {
+ if(op.getPivot()!=-1) {
+ if (op.getPivot() < 0 || op.getPivot() >= expressionPlans.get(0).size()) {
+ FrontendException fe = new FrontendException("PIVOT is out of bound");
+ throw fe;
+ }
+ }
+ else
+ op.setPivot((int)(Math.round(expressionPlans.get(0).size()/2.0)));
+ } catch (FrontendException e) {
+ throw new ParserValidationException(intStream, loc, e);
+ }
+
// check if continuously occurring cube operations be combined
combineCubeOperations((ArrayList<String>) operations, expressionPlans);
@@ -713,6 +733,7 @@ public class LogicalPlanBuilder {
// build group by operator
try {
+ groupby.setPivot(op.getPivot());
return buildGroupOp(loc, (LOCogroup) groupby, op.getAlias(), inpAliases, exprPlansCopy,
GROUPTYPE.REGULAR, innerFlags, null);
} catch (ParserValidationException pve) {
Modified: pig/branches/spark/src/org/apache/pig/parser/LogicalPlanGenerator.g
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/parser/LogicalPlanGenerator.g?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/parser/LogicalPlanGenerator.g (original)
+++ pig/branches/spark/src/org/apache/pig/parser/LogicalPlanGenerator.g Tue Jan 27 02:27:45 2015
@@ -493,12 +493,20 @@ func_args returns[List<String> args]
// It also outputs the order of operations i.e in this case CUBE operation followed by ROLLUP operation
// These inputs are passed to buildCubeOp methods which then builds the logical plan for CUBE operator.
// If user specifies STAR or RANGE expression for dimensions then it will be expanded inside buildCubeOp.
+pivot_clause returns[int pivot]
+ : ^( PIVOT INTEGER )
+ {
+ $pivot = Integer.parseInt( $INTEGER.text );
+ }
+;
+
cube_clause returns[String alias]
scope {
LOCube cubeOp;
MultiMap<Integer, LogicalExpressionPlan> cubePlans;
List<String> operations;
int inputIndex;
+ int pivot;
}
scope GScope;
@init {
@@ -548,7 +556,7 @@ cube_rollup_list returns[String operatio
@init {
$plans = new ArrayList<LogicalExpressionPlan>();
}
- : ^( ( CUBE { $operation = "CUBE"; } | ROLLUP { $operation = "ROLLUP"; } ) cube_by_expr_list { $plans = $cube_by_expr_list.plans; } )
+ : ^( CUBE { $operation = "CUBE"; } cube_by_expr_list { $plans = $cube_by_expr_list.plans; } ) | ^( ROLLUP { $operation = "ROLLUP"; } cube_by_expr_list { $plans = $cube_by_expr_list.plans; } pivot_clause? { if ($pivot_clause.tree!=null) builder.setPivotRollupCubeOp($cube_clause::cubeOp, $pivot_clause.pivot); } )
;
cube_by_expr_list returns[List<LogicalExpressionPlan> plans]
@@ -1941,6 +1949,7 @@ eid returns[String id] : rel_str_op { $i
| COGROUP { $id = $COGROUP.text; }
| CUBE { $id = $CUBE.text; }
| ROLLUP { $id = $ROLLUP.text; }
+ | PIVOT { $id = $PIVOT.text; }
| JOIN { $id = $JOIN.text; }
| CROSS { $id = $CROSS.text; }
| UNION { $id = $UNION.text; }
Modified: pig/branches/spark/src/org/apache/pig/parser/QueryLexer.g
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/parser/QueryLexer.g?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/parser/QueryLexer.g (original)
+++ pig/branches/spark/src/org/apache/pig/parser/QueryLexer.g Tue Jan 27 02:27:45 2015
@@ -153,6 +153,9 @@ ONSCHEMA : 'ONSCHEMA'
PARALLEL : 'PARALLEL'
;
+PIVOT : 'PIVOT'
+;
+
PARTITION : 'PARTITION'
;
Modified: pig/branches/spark/src/org/apache/pig/parser/QueryParser.g
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/parser/QueryParser.g?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/parser/QueryParser.g (original)
+++ pig/branches/spark/src/org/apache/pig/parser/QueryParser.g Tue Jan 27 02:27:45 2015
@@ -595,7 +595,10 @@ union_clause : UNION^ ONSCHEMA? rel_list
cube_clause : CUBE rel BY cube_rollup_list ( COMMA cube_rollup_list )* -> ^( CUBE rel ^( BY cube_rollup_list+ ) )
;
-cube_rollup_list : ( CUBE | ROLLUP )^ LEFT_PAREN! real_arg ( COMMA! real_arg )* RIGHT_PAREN!
+cube_rollup_list : ( CUBE^ LEFT_PAREN! real_arg ( COMMA! real_arg )* RIGHT_PAREN! ) | ( ROLLUP^ LEFT_PAREN! real_arg ( COMMA! real_arg )* RIGHT_PAREN! pivot_clause? )
+;
+
+pivot_clause : PIVOT^ INTEGER
;
flatten_clause : FLATTEN^ LEFT_PAREN! expr RIGHT_PAREN!
Modified: pig/branches/spark/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ pig/branches/spark/src/org/apache/pig/tools/grunt/GruntParser.java Tue Jan 27 02:27:45 2015
@@ -451,12 +451,16 @@ public class GruntParser extends PigScri
@Override
protected void processRegister(String jar) throws IOException {
filter.validate(PigCommandFilter.Command.REGISTER);
+ jar = parameterSubstitutionInGrunt(jar);
mPigServer.registerJar(jar);
}
@Override
protected void processRegister(String path, String scriptingLang, String namespace) throws IOException, ParseException {
filter.validate(PigCommandFilter.Command.REGISTER);
+ path = parameterSubstitutionInGrunt(path);
+ scriptingLang = parameterSubstitutionInGrunt(scriptingLang);
+ namespace = parameterSubstitutionInGrunt(namespace);
if(path.endsWith(".jar")) {
if(scriptingLang != null || namespace != null) {
throw new ParseException("Cannot register a jar with a scripting language or namespace");
@@ -568,6 +572,8 @@ public class GruntParser extends PigScri
@Override
protected void processSet(String key, String value) throws IOException, ParseException {
filter.validate(PigCommandFilter.Command.SET);
+ key = parameterSubstitutionInGrunt(key);
+ value = parameterSubstitutionInGrunt(value);
if (key.equals("debug"))
{
if (value.equals("on"))
@@ -636,6 +642,7 @@ public class GruntParser extends PigScri
@Override
protected void processCat(String path) throws IOException {
filter.validate(PigCommandFilter.Command.CAT);
+ path = parameterSubstitutionInGrunt(path);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -685,6 +692,7 @@ public class GruntParser extends PigScri
@Override
protected void processCD(String path) throws IOException {
filter.validate(PigCommandFilter.Command.CD);
+ path = parameterSubstitutionInGrunt(path);
ContainerDescriptor container;
if(mExplain == null) { // process only if not in "explain" mode
@@ -820,6 +828,7 @@ public class GruntParser extends PigScri
@Override
protected void processLS(String path) throws IOException {
filter.validate(PigCommandFilter.Command.LS);
+ path = parameterSubstitutionInGrunt(path);
if (mExplain == null) { // process only if not in "explain" mode
@@ -945,6 +954,8 @@ public class GruntParser extends PigScri
protected void processMove(String src, String dst) throws IOException
{
filter.validate(PigCommandFilter.Command.MV);
+ src = parameterSubstitutionInGrunt(src);
+ dst = parameterSubstitutionInGrunt(dst);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -971,6 +982,8 @@ public class GruntParser extends PigScri
protected void processCopy(String src, String dst) throws IOException
{
filter.validate(PigCommandFilter.Command.CP);
+ src = parameterSubstitutionInGrunt(src);
+ dst = parameterSubstitutionInGrunt(dst);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -993,6 +1006,8 @@ public class GruntParser extends PigScri
protected void processCopyToLocal(String src, String dst) throws IOException
{
filter.validate(PigCommandFilter.Command.COPYTOLOCAL);
+ src = parameterSubstitutionInGrunt(src);
+ dst = parameterSubstitutionInGrunt(dst);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -1015,6 +1030,8 @@ public class GruntParser extends PigScri
protected void processCopyFromLocal(String src, String dst) throws IOException
{
filter.validate(PigCommandFilter.Command.COPYFROMLOCAL);
+ src = parameterSubstitutionInGrunt(src);
+ dst = parameterSubstitutionInGrunt(dst);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -1037,6 +1054,7 @@ public class GruntParser extends PigScri
protected void processMkdir(String dir) throws IOException
{
filter.validate(PigCommandFilter.Command.MKDIR);
+ dir = parameterSubstitutionInGrunt(dir);
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -1052,6 +1070,7 @@ public class GruntParser extends PigScri
protected void processPig(String cmd) throws IOException
{
int start = 1;
+ cmd = parameterSubstitutionInGrunt(cmd);
if (!mInteractive) {
start = getLineNumber();
}
@@ -1068,6 +1087,7 @@ public class GruntParser extends PigScri
protected void processRemove(String path, String options) throws IOException {
filter.validate(PigCommandFilter.Command.RM);
filter.validate(PigCommandFilter.Command.RMF);
+ path = parameterSubstitutionInGrunt(path);
int MAX_MS_TO_WAIT_FOR_FILE_DELETION = 10 * 60 * 1000;
int MS_TO_SLEEP_WHILE_WAITING_FOR_FILE_DELETION = 250;
@@ -1112,6 +1132,9 @@ public class GruntParser extends PigScri
@Override
protected void processFsCommand(String[] cmdTokens) throws IOException {
filter.validate(PigCommandFilter.Command.FS);
+ for (int i = 0 ; i < cmdTokens.length ; i++) {
+ cmdTokens[i] = parameterSubstitutionInGrunt(cmdTokens[i]);
+ }
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
@@ -1138,6 +1161,9 @@ public class GruntParser extends PigScri
@Override
protected void processShCommand(String[] cmdTokens) throws IOException {
filter.validate(PigCommandFilter.Command.SH);
+ for (int i = 0 ; i < cmdTokens.length ; i++) {
+ cmdTokens[i] = parameterSubstitutionInGrunt(cmdTokens[i]);
+ }
if(mExplain == null) { // process only if not in "explain" mode
try {
executeBatch();
@@ -1241,6 +1267,7 @@ public class GruntParser extends PigScri
@Override
protected void processSQLCommand(String cmd) throws IOException{
+ cmd = parameterSubstitutionInGrunt(cmd);
if(mExplain == null) { // process only if not in "explain" mode
if (!mPigServer.getPigContext().getProperties().get("pig.sql.type").equals("hcat")) {
throw new IOException("sql command only support hcat currently");
@@ -1259,6 +1286,22 @@ public class GruntParser extends PigScri
}
}
+ @Override
+ protected void processDefault(String key, String value) throws IOException {
+ parameterSubstitutionInGrunt("%default " + key + " " + value);
+ }
+
+ @Override
+ protected void processDeclare(String key, String value) throws IOException {
+ parameterSubstitutionInGrunt("%declare " + key + " " + value);
+ }
+
+ private String parameterSubstitutionInGrunt(String input) throws IOException {
+ if (mInteractive && input != null) {
+ return mPigServer.getPigContext().doParamSubstitution(new BufferedReader(new StringReader(input))).trim();
+ }
+ return input;
+ }
/**
* StreamPrinter.
*
Modified: pig/branches/spark/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original)
+++ pig/branches/spark/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Tue Jan 27 02:27:45 2015
@@ -120,6 +120,10 @@ public abstract class PigScriptParser
abstract protected void processScript(String script, boolean batch, List<String> params, List<String> files) throws IOException, ParseException;
+ abstract protected void processDefault(String key, String value) throws IOException;
+
+ abstract protected void processDeclare(String key, String value) throws IOException;
+
static String unquote(String s)
{
if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
@@ -186,6 +190,8 @@ TOKEN: {<XML: "-xml">}
TOKEN: {<OUT: "-out">}
TOKEN: {<BRIEF: "-brief">}
TOKEN: {<N: "-n">}
+TOKEN: {<PIGDEFAULT: "%default" >}
+TOKEN: {<DECLARE: "%declare" >}
// internal use commands
TOKEN: {<SCRIPT_DONE: "scriptDone">}
@@ -420,6 +426,7 @@ TOKEN : { <QUOTEDSTRING : "'"
)
)*
"'"> }
+TOKEN: {<SHELLCMD: "`" (~["`"])* "`" >}
void parse() throws IOException:
{
Token t1, t2, t3;
@@ -598,6 +605,16 @@ void parse() throws IOException:
{processSet();}
)
|
+ <PIGDEFAULT>
+ t1 = GetKey()
+ t2 = GetDefaultValue()
+ {processDefault(t1.image, t2.image);}
+ |
+ <DECLARE>
+ t1 = GetKey()
+ t2 = GetDefaultValue()
+ {processDeclare(t1.image, t2.image);}
+ |
<EOF>
{quit();}
|
@@ -796,6 +813,22 @@ Token GetValue() :
)
{return t;}
+}
+
+Token GetDefaultValue() :
+{
+ Token t;
+}
+{
+ (
+ t = GetPath()
+ |
+ t = <QUOTEDSTRING>
+ |
+ t = <SHELLCMD>
+ )
+
+ {return t;}
}
Token GetReserved () :
Modified: pig/branches/spark/src/org/apache/pig/tools/pigstats/tez/TezScriptState.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/tools/pigstats/tez/TezScriptState.java?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/tools/pigstats/tez/TezScriptState.java (original)
+++ pig/branches/spark/src/org/apache/pig/tools/pigstats/tez/TezScriptState.java Tue Jan 27 02:27:45 2015
@@ -203,7 +203,7 @@ public class TezScriptState extends Scri
return dagScriptInfo.get(dagName);
}
- static class TezDAGScriptInfo {
+ public static class TezDAGScriptInfo {
private static final Log LOG = LogFactory.getLog(TezDAGScriptInfo.class);
private TezOperPlan tezPlan;
Modified: pig/branches/spark/test/e2e/harness/build.xml
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/harness/build.xml?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/harness/build.xml (original)
+++ pig/branches/spark/test/e2e/harness/build.xml Tue Jan 27 02:27:45 2015
@@ -41,6 +41,7 @@
<include name="Log.pm"/>
<include name="Properties.pm"/>
<include name="test_harness.pl"/>
+ <include name="xmlReport.pl"/>
</fileset>
</copy>
Modified: pig/branches/spark/test/e2e/harness/xmlReport.pl
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/harness/xmlReport.pl?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/harness/xmlReport.pl (original)
+++ pig/branches/spark/test/e2e/harness/xmlReport.pl Tue Jan 27 02:27:45 2015
@@ -15,7 +15,7 @@ unshift( @INC, ".");
########################################################################
# Sub: testcase
# Prints the result for a test case.
-#
+#
# Parameters:
# $class - the test file
# $test - the test name
@@ -24,115 +24,122 @@ unshift( @INC, ".");
# Returns:
# None
#
-sub getTestcase($$$$$$){
-
- my ( $class_name, $group_name, $test_name, $time, $status, $msg )=@_;
- my $result = "";
-
- if ( $status =~ "passed" ){
- $result = "\n<testcase classname=\"$class_name\" name=\"$test_name\" time=\"$time\"/>";
+sub getTestcase($$$$$){
- } elsif ( $status =~ "skipped" ){
- $result = "\n<testcase classname=\"$class_name\" name=\"$test_name\" time=\"0\">"
- . "\n<skipped type=\"$status\">$msg</skipped>"
- . "\n</testcase>"
- ;
- } else {
- $result = "\n<testcase classname=\"$class_name\" name=\"$test_name\" time=\"$time\">"
- . "\n<error type=\"$status\">$msg</error>"
- . "\n</testcase>"
- ;
- } return $result;
+ my ( $exec_type, $test_name, $time, $status, $msg )=@_;
+ my $result = "";
+ my $group_name = substr ($test_name, 0, rindex ($test_name, "_"));
+
+ if ( $status =~ "passed" ){
+ $result = "\n<testcase classname=\"$exec_type.$group_name\" name=\"$test_name\" time=\"$time\"/>";
+ } elsif ( $status =~ "skipped" ){
+ $result = "\n<testcase classname=\"$exec_type.$group_name\" name=\"$test_name\" time=\"0\">"
+ . "\n<skipped type=\"$status\">$msg</skipped>"
+ . "\n</testcase>"
+ ;
+ } else {
+ $result = "\n<testcase classname=\"$exec_type.$group_name\" name=\"$test_name\" time=\"$time\">"
+ . "\n<error type=\"$status\">$msg</error>"
+ . "\n</testcase>"
+ ;
+ }
+ return $result;
}
-sub printXmlReport($) {
+sub printXmlReport($$) {
- my $host = Sys::Hostname::hostname();
- my $tmpFileName = shift;
- my $reportFileName = shift;
-
- my $testcases = "";
- my $total_time= 0;
- my $testnameSuffix = "";
-
- if ($tmpFileName =~ m/-local/) {
- $testnameSuffix = "_local";
- }
-
- my $passedCount = 0;
- my $failureCount = 0;
- my $errorCount = 0;
- my $skippedCount = 0;
- my $totalCount = 0;
-
- my %test2starttime = { '', 0 };
-
- open( IN, "$tmpFileName")
- || die "Could not open $tmpFileName\n";
-
- while(<IN>){
-
- my $line = $_;
- # e.g.: "Beginning test Checkin_1 at 1346855793"
- if ( $line =~ m/Beginning test (.*) at (.*)/ ) {
- # put "test - start time" pair to the hash:
- $test2starttime{ $1 } = $2;
- } else {
- if ( $line =~ "TestDriver::run" ) {
- my $duration = 0;
- if ( $line =~ m/TestDriver::run.*Failed to run test (.*) <(.*)/ ) {
- # ERROR TestDriver::run at : 470 Failed to run test ClassResolution_1 <Failed running ./out/pigtest/hadoopqa/hadoopqa.1327755958/ClassResolution_1_benchmark.pig
- #print "test aborted: $line";
- $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix, 0, "aborted", $2 );
- $errorCount++;
- } elsif ( $line =~ m/TestDriver::run.*Test (.*) SUCCEEDED at (.*)/ ) {
- # INFO: TestDriver::run() at 444:Test Unicode_cmdline_1 SUCCEEDED at 1327751873
- #print "test passed: $1 $2 line: $line";
- $passedCount++;
- $duration = $2 - $test2starttime{ $1 };
- $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix, $duration, "passed", "" );
- } elsif ( $line =~ m/TestDriver::run.*Test (.*) FAILED at (.*)/ ) {
- $failureCount++;
- $duration = $2 - $test2starttime{ $1 };
- $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix, $duration, "failed", "" );
- #print "test failed: $1 $2 line: $line";
- } elsif ($line =~ "Running TEST GROUP") {
- next;
- } elsif ($line =~ m/TestDriver::run.*Test (.*) SKIPPED at (.*)/) {
- # INFO: TestDriver::run() at 444:Test StreamingLocalErrors_1 SKIPPED at 1327923441
- $skippedCount++;
- $duration = $2 - $test2starttime{ $1 };
- $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix, $duration, "skipped", "" );
- } else {
- print STDERR "Ignored line: $line";
- next;
- }
- #$total_time= $total_time + $time;
- $totalCount++;
- }
- }
-
- }
- close(IN);
- #Report
- my $host = Sys::Hostname::hostname();
- my $run_name = "e2e tests";
- my $report=
- '<?xml version="1.0" encoding="UTF-8" ?>'
- . "\n<testsuite errors=\"$errorCount\" failures=\"$failureCount\" skips=\"$skippedCount\" hostname=\"$host\" name=\"$run_name\" tests=\"$totalCount\" time=\"$total_time\">"
- . "\n$testcases"
- . "\n</testsuite>"
- . "\n";
- print $report;
+ my $host = Sys::Hostname::hostname();
+ my $reportFileName = shift;
+ my $execType = shift;
+
+ my $testcases = "";
+ my $testRunStartTime = 0;
+ my $testRunEndTime = 0;
+
+ my $passedCount = 0;
+ my $failureCount = 0;
+ my $errorCount = 0;
+ my $skippedCount = 0;
+ my $totalCount = 0;
+
+ my %test2starttime = { '', 0 };
+
+ open( IN, "$reportFileName")
+ || die "Could not open $reportFileName\n";
+
+ while(<IN>) {
+
+ my $line = $_;
+ if ($line =~ m/Beginning test run at (.*)/ ) {
+ # e.g: Beginning test run at 1416422503
+ $testRunStartTime = $1;
+ } elsif ($line =~ m/Finished test run at (.*)/ ) {
+ # e.g: Finished test run at 1416428021
+ $testRunEndTime = $1;
+ } elsif ( $line =~ m/Beginning test (.*) at (.*)/ ) {
+ # e.g.: "Beginning test Checkin_1 at 1346855793"
+ # put "test - start time" pair to the hash:
+ $test2starttime{ $1 } = $2;
+ } elsif ( $line =~ "TestDriver::run" ) {
+ my $duration = 0;
+ if ( $line =~ m/TestDriver::run.*Failed to run test (.*) <(.*)/ ) {
+ # ERROR TestDriver::run at : 470 Failed to run test ClassResolution_1 <Failed running ./out/pigtest/hadoopqa/hadoopqa.1327755958/ClassResolution_1_benchmark.pig
+ #print "test aborted: $line";
+ $testcases .= getTestcase ( $execType, $1 , 0, "aborted", $2 );
+ $errorCount++;
+ } elsif ( $line =~ m/TestDriver::run.*Test (.*) SUCCEEDED at (.*)/ ) {
+ # INFO: TestDriver::run() at 444:Test Unicode_cmdline_1 SUCCEEDED at 1327751873
+ #print "test passed: $1 $2 line: $line";
+ $passedCount++;
+ $duration = $2 - $test2starttime{ $1 };
+ $testcases .= getTestcase ( $execType, $1 , $duration, "passed", "" );
+ } elsif ( $line =~ m/TestDriver::run.*Test (.*) FAILED at (.*)/ ) {
+ $failureCount++;
+ $duration = $2 - $test2starttime{ $1 };
+ $testcases .= getTestcase ( $execType, $1 , $duration, "failed", "" );
+ #print "test failed: $1 $2 line: $line";
+ } elsif ($line =~ "Running TEST GROUP") {
+ next;
+ } elsif ($line =~ m/TestDriver::run.*Test (.*) SKIPPED at (.*)/) {
+ # INFO: TestDriver::run() at 444:Test StreamingLocalErrors_1 SKIPPED at 1327923441
+ $skippedCount++;
+ $duration = $2 - $test2starttime{ $1 };
+ $testcases .= getTestcase ( $execType, $1 , $duration, "skipped", "" );
+ } else {
+ print STDERR "Ignored line: $line";
+ next;
+ }
+ $totalCount++;
+ }
+
+ }
+ close(IN);
+ #Report
+ my $host = Sys::Hostname::hostname();
+ my $run_name = $execType . " e2e tests";
+ my $total_time = $testRunEndTime - $testRunStartTime;
+ my $report= '<?xml version="1.0" encoding="UTF-8" ?>'
+ . "\n<testsuite errors=\"$errorCount\" failures=\"$failureCount\" skips=\"$skippedCount\" hostname=\"$host\" name=\"$run_name\" tests=\"$totalCount\" time=\"$total_time\">"
+ . "\n$testcases"
+ . "\n</testsuite>"
+ . "\n";
+ print $report;
}
if (!defined( $ARGV[0] )) {
- die "No input log file specified\n";
+ die "No input log file specified\n";
+}
+
+my $execType = $ARGV[1];
+if (!defined( $ARGV[1] )) {
+ print stderr "No execution type specified. Using mapred as default\n";
+ $execType = "mapred";
}
-printXmlReport($ARGV[0]);
+# Harness log file name, exec type which will be used as test group name
+printXmlReport($ARGV[0], $execType);
Modified: pig/branches/spark/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/build.xml?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/build.xml (original)
+++ pig/branches/spark/test/e2e/pig/build.xml Tue Jan 27 02:27:45 2015
@@ -305,6 +305,7 @@
<arg value="${test.location}/tests/macro.conf"/>
<arg value="${test.location}/tests/orc.conf"/>
<arg value="${test.location}/tests/hcat.conf"/>
+ <arg value="${test.location}/tests/utf8.conf"/>
</exec>
</target>
Modified: pig/branches/spark/test/e2e/pig/deployers/ExistingClusterDeployer.pm
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/deployers/ExistingClusterDeployer.pm?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/deployers/ExistingClusterDeployer.pm (original)
+++ pig/branches/spark/test/e2e/pig/deployers/ExistingClusterDeployer.pm Tue Jan 27 02:27:45 2015
@@ -240,6 +240,16 @@ sub generateData
'filetype' => "ranking",
'rows' => 30,
'hdfs' => "singlefile/prerank",
+ }, {
+ 'name' => "utf8Voter",
+ 'filetype' => "utf8Voter",
+ 'rows' => 30,
+ 'hdfs' => "utf8Data/éæ°/utf8Voter",
+ }, {
+ 'name' => "utf8Student",
+ 'filetype' => "utf8Student",
+ 'rows' => 300,
+ 'hdfs' => "utf8Data/å¦ç/utf8Student",
}
);
Modified: pig/branches/spark/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/drivers/TestDriverPig.pm?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/branches/spark/test/e2e/pig/drivers/TestDriverPig.pm Tue Jan 27 02:27:45 2015
@@ -277,12 +277,7 @@ sub runPigCmdLine
# Build the command
my @baseCmd = $self->getPigCmd($testCmd, $log);
- if ($testCmd->{'exectype'} eq "tez") {
- push(@baseCmd, ("-x", "tez"));
- }
- if ($testCmd->{'exectype'} eq "spark") {
- push(@baseCmd, ("-x", "spark"));
- }
+ push(@baseCmd, ("-x", $testCmd->{'exectype'}));
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs
@@ -440,9 +435,9 @@ sub getPigCmd($$$)
$additionalJavaParams .= " -Dmapred.local.dir=$hadoopTmpDir -Dmapreduce.cluster.local.dir=$hadoopTmpDir";
}
TestDriver::dbg("Additional java parameters: [$additionalJavaParams].\n");
-
- push(@pigCmd, ("-x", "local"));
}
+
+ push(@pigCmd, ("-x", $testCmd->{'exectype'}));
if (defined($testCmd->{'java_params'}) || defined($additionalJavaParams)) {
if (defined($testCmd->{'java_params'})) {
@@ -506,12 +501,7 @@ sub runPig
# Build the command
my @baseCmd = $self->getPigCmd($testCmd, $log);
- if ($testCmd->{'exectype'} eq "tez") {
- push(@baseCmd, ("-x", "tez"));
- }
- if ($testCmd->{'exectype'} eq "spark") {
- push(@baseCmd, ("-x", "spark"));
- }
+ push(@baseCmd, ("-x", $testCmd->{'exectype'}));
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs
Modified: pig/branches/spark/test/e2e/pig/tests/bigdata.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/bigdata.conf?rev=1654955&r1=1654954&r2=1654955&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/bigdata.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/bigdata.conf Tue Jan 27 02:27:45 2015
@@ -92,7 +92,7 @@ store c into ':OUTPATH:';\,
]
},
{
- 'name' => 'BigData_Stream',
+ 'name' => 'BigData_Streaming',
'tests' => [
{
'num' => 1,