You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/05/20 23:41:07 UTC
svn commit: r658450 - in /incubator/pig/branches/types:
src/org/apache/pig/impl/logicalLayer/
src/org/apache/pig/impl/logicalLayer/parser/
src/org/apache/pig/impl/logicalLayer/validators/ test/org/apache/pig/test/
Author: gates
Date: Tue May 20 14:41:07 2008
New Revision: 658450
URL: http://svn.apache.org/viewvc?rev=658450&view=rev
Log:
PIG-159 Checked in Santhosh's changes to fix a few bugs in LOCross, LOUnion, change INTEGER keyword to INT.
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java Tue May 20 14:41:07 2008
@@ -64,13 +64,12 @@
@Override
public Schema getSchema() throws FrontendException {
if (!mIsSchemaComputed && (null == mSchema)) {
- Collection<LogicalOperator> pred = mPlan.getPredecessors(this);
List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>();
Map<Schema.FieldSchema, String> flattenAlias = new HashMap<Schema.FieldSchema, String>();
Map<String, Boolean> inverseFlattenAlias = new HashMap<String, Boolean>();
Map<String, Integer> aliases = new HashMap<String, Integer>();
- for (LogicalOperator op : pred) {
+ for (LogicalOperator op : mInputs) {
String opAlias = op.getAlias();
Schema s = op.getSchema();
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java Tue May 20 14:41:07 2008
@@ -33,7 +33,7 @@
public class LOSplit extends LogicalOperator {
private static final long serialVersionUID = 2L;
- private Map<String, LogicalPlan> mCondPlans;
+ //private Map<String, LogicalPlan> mCondPlans;
private ArrayList<LogicalOperator> mOutputs;
private static Log log = LogFactory.getLog(LOSplit.class);
@@ -48,17 +48,17 @@
* list of conditions for the split
*/
public LOSplit(LogicalPlan plan, OperatorKey key,
- ArrayList<LogicalOperator> outputs,
- Map<String, LogicalPlan> condPlans) {
+ ArrayList<LogicalOperator> outputs) {
+ // Map<String, LogicalPlan> condPlans) {
super(plan, key);
mOutputs = outputs;
- mCondPlans = condPlans;
+ //mCondPlans = condPlans;
}
public List<LogicalOperator> getOutputs() {
return mOutputs;
}
-
+/*
public Collection<LogicalPlan> getConditionPlans() {
return mCondPlans.values();
}
@@ -70,7 +70,7 @@
public void addOutputAlias(String output, LogicalPlan cond) {
mCondPlans.put(output, cond);
}
-
+*/
public void addOutput(LogicalOperator lOp) {
mOutputs.add(lOp);
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java Tue May 20 14:41:07 2008
@@ -32,6 +32,7 @@
private static final long serialVersionUID = 2L;
protected int mIndex;
+ private LogicalPlan mCondPlan;
/**
* @param plan
@@ -43,9 +44,14 @@
* @param conditions
* list of conditions for the split
*/
- public LOSplitOutput(LogicalPlan plan, OperatorKey key, int index) {
+ public LOSplitOutput(LogicalPlan plan, OperatorKey key, int index, LogicalPlan condPlan) {
super(plan, key);
this.mIndex = index;
+ this.mCondPlan = condPlan;
+ }
+
+ public LogicalPlan getConditionPlan() {
+ return mCondPlan;
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java Tue May 20 14:41:07 2008
@@ -64,19 +64,16 @@
Iterator<LogicalOperator> iter = s.iterator();
LogicalOperator op = iter.next();
if (null == op) {
- log.debug("getSchema: Operator not in plan");
throw new FrontendException("Could not find operator in plan");
}
mSchema = op.getSchema();
- log.debug("Printing aliases");
- mSchema.printAliases();
while(iter.hasNext()) {
op = iter.next();
- if(null == mSchema) {
- log.debug("Schema is null, cannot perform schema merge");
- throw new FrontendException("Schema is null, cannot perform schema merge");
+ if(null != mSchema) {
+ mSchema = mSchema.merge(op.getSchema(), false);
+ } else {
+ mSchema = op.getSchema();
}
- mSchema = mSchema.merge(op.getSchema(), false);
}
mIsSchemaComputed = true;
} catch (FrontendException fe) {
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java Tue May 20 14:41:07 2008
@@ -194,15 +194,8 @@
*/
protected void visit(LOSplit split) throws VisitorException {
// Visit each of split's conditions
- for(LogicalPlan lp: split.getConditionPlans()) {
- if (null != lp) {
- PlanWalker w = new DependencyOrderWalker(lp);
- pushWalker(w);
- for(LogicalOperator logicalOp: lp.getRoots()) {
- logicalOp.visit(this);
- }
- popWalker();
- }
+ for(LogicalOperator logicalOp: split.getOutputs()) {
+ logicalOp.visit(this);
}
}
@@ -295,7 +288,15 @@
}
protected void visit(LOSplitOutput sop) throws VisitorException {
-
+ LogicalPlan lp = sop.getConditionPlan();
+ if (null != lp) {
+ PlanWalker w = new DependencyOrderWalker(lp);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: lp.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
}
protected void visit(LODistinct dt) throws VisitorException {
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Tue May 20 14:41:07 2008
@@ -268,9 +268,9 @@
}
void addSplitOutput(LogicalPlan lp, LOSplit splitOp, String alias, LogicalPlan condPlan, int index) throws PlanException{
- LogicalOperator splitOutput = new LOSplitOutput(lp, new OperatorKey(scope, getNextId()), index);
+ LogicalOperator splitOutput = new LOSplitOutput(lp, new OperatorKey(scope, getNextId()), index, condPlan);
splitOp.addOutput(splitOutput);
- splitOp.addOutputAlias(alias, condPlan);
+ //splitOp.addOutputAlias(alias, condPlan);
addAlias(alias, splitOutput);
lp.add(splitOutput);
@@ -383,7 +383,7 @@
TOKEN : { <EVAL : "eval"> }
TOKEN : { <ASC : "asc"> }
TOKEN : { <DESC : "desc"> }
-TOKEN : { <INT : "integer"> }
+TOKEN : { <INT : "int"> }
TOKEN : { <LONG : "long"> }
TOKEN : { <FLOAT : "float"> }
TOKEN : { <DOUBLE : "double"> }
@@ -435,10 +435,7 @@
)
{
if(null != root) {
- if(null != root.getSchema()) {
- log.debug("Root: " + root.getClass().getName() + " schema aliases");
- root.getSchema().printAliases();
- }
+ log.debug("Root: " + root.getClass().getName() + " schema: " + root.getSchema());
}
log.trace("Exiting Parse");
@@ -460,7 +457,7 @@
(
input = NestedExpr(lp) <INTO>
{
- splitOp = new LOSplit(lp, input.getOperatorKey(), new ArrayList<LogicalOperator>(), new HashMap<String,LogicalPlan>());
+ splitOp = new LOSplit(lp, input.getOperatorKey(), new ArrayList<LogicalOperator>());
lp.add(splitOp);
log.debug("Adding operator " + splitOp.getClass().getName() + " to the logical plan");
lp.connect(input, splitOp);
@@ -1863,9 +1860,13 @@
{
(t = <INT>| t = <LONG>| t = <FLOAT>| t = <DOUBLE>| t = <CHARARRAY>| t = <BYTEARRAY>)
{
+ String typeName = t.image;
log.debug(" nameToTypeMap: " + nameToTypeMap);
- log.debug("t: " + t + " type: " + nameToTypeMap.get(t.image.toUpperCase()));
- type = nameToTypeMap.get(t.image.toUpperCase());
+ log.debug("t: " + t + " type: " + nameToTypeMap.get(typeName.toUpperCase()));
+ if(typeName.equalsIgnoreCase("int")) {
+ typeName = "integer";
+ }
+ type = nameToTypeMap.get(typeName.toUpperCase());
log.trace("Exiting BasicType");
return type;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java Tue May 20 14:41:07 2008
@@ -856,8 +856,29 @@
throw new AssertionError("LOSplitOutput can only have 1 input") ;
}
+ LogicalOperator input = list.get(0);
+ LogicalPlan condPlan = op.getConditionPlan() ;
+
+ // Check that the inner plan has only 1 output port
+ if (!condPlan.isSingleLeafPlan()) {
+ String msg = "Split's cond plan can only have one output (leaf)" ;
+ msgCollector.collect(msg, MessageType.Error) ;
+ throw new VisitorException(msg) ;
+ }
+
+ checkInnerPlan(condPlan, input) ;
+
+ byte innerCondType = condPlan.getLeaves().get(0).getType() ;
+ if (innerCondType != DataType.BOOLEAN) {
+ String msg = "Split's condition must evaluate to boolean" ;
+ msgCollector.collect(msg, MessageType.Error) ;
+ throw new VisitorException(msg) ;
+ }
+
+ op.setType(input.getType()) ; // This should be bag always
+
try {
- op.setSchema(list.get(0).getSchema()) ;
+ op.setSchema(input.getSchema()) ;
}
catch (ParseException pe) {
String msg = "Problem while reading schemas from"
@@ -1083,32 +1104,6 @@
LogicalOperator input = inputList.get(0) ;
- // Checking internal plans.
- ArrayList<LogicalPlan> condPlans
- = new ArrayList<LogicalPlan>(split.getConditionPlans()) ;
-
- for(int i=0;i < split.getConditionPlans().size(); i++) {
-
- LogicalPlan condPlan = condPlans.get(i) ;
-
- // Check that the inner plan has only 1 output port
- if (!condPlan.isSingleLeafPlan()) {
- String msg = "Split's cond plan can only have one output (leaf)" ;
- msgCollector.collect(msg, MessageType.Error) ;
- throw new VisitorException(msg) ;
- }
-
- checkInnerPlan(condPlan, input) ;
-
- byte innerCondType = condPlan.getLeaves().get(0).getType() ;
- if (innerCondType != DataType.BOOLEAN) {
- String msg = "Split's condition must evaluate to boolean" ;
- msgCollector.collect(msg, MessageType.Error) ;
- throw new VisitorException(msg) ;
- }
-
- }
-
split.setType(input.getType()) ; // This should be bag always
try {
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java Tue May 20 14:41:07 2008
@@ -742,17 +742,17 @@
@Test
public void testQuery64() {
buildPlan("a = load 'a' as (name: chararray, details: tuple(age, gpa), mymap: map[]);");
- buildPlan("c = load 'a' as (name, details: bag{mytuple: tuple(age: integer, gpa)});");
+ buildPlan("c = load 'a' as (name, details: bag{mytuple: tuple(age: int, gpa)});");
buildPlan("b = group a by details;");
String query = "d = foreach b generate group.age;";
buildPlan(query);
buildPlan("e = foreach a generate name, details;");
- buildPlan("f = LOAD 'myfile' AS (garage: bag{tuple1: tuple(num_tools: integer)}, links: bag{tuple2: tuple(websites: chararray)}, page: bag{something_stupid: tuple(yeah_double: double)}, coordinates: bag{another_tuple: tuple(ok_float: float, bite_the_array: bytearray, bag_of_unknown: bag{})});");
+ buildPlan("f = LOAD 'myfile' AS (garage: bag{tuple1: tuple(num_tools: int)}, links: bag{tuple2: tuple(websites: chararray)}, page: bag{something_stupid: tuple(yeah_double: double)}, coordinates: bag{another_tuple: tuple(ok_float: float, bite_the_array: bytearray, bag_of_unknown: bag{})});");
}
@Test
public void testQueryFail64() {
- String query = "foreach (load 'myfile' as (col1, col2 : bag{age: integer})) generate col1 ;";
+ String query = "foreach (load 'myfile' as (col1, col2 : bag{age: int})) generate col1 ;";
try {
buildPlan(query);
} catch (AssertionFailedError e) {
@@ -782,14 +782,14 @@
@Test
public void testQuery66() {
- buildPlan("define myFunc = com.mycompany.myudfs.myfunc(integer i, chararray c);");
+ buildPlan("define myFunc = com.mycompany.myudfs.myfunc(int i, chararray c);");
buildPlan("a = load 'input1' as (name, age, gpa);");
buildPlan("b = foreach a generate myFunc($0, $1);");
}
@Test
public void testQueryFail66() {
- buildPlan("define myFunc = com.mycompany.myudfs.myfunc(integer i, chararray c);");
+ buildPlan("define myFunc = com.mycompany.myudfs.myfunc(int i, chararray c);");
buildPlan("a = load 'input1' as (name, age, gpa);");
try {
buildPlan("b = foreach a generate myFunc($0, $1, $2);");
@@ -853,9 +853,9 @@
@Test
public void testQuery72() {
buildPlan("split (load 'a') into x if $0 > '7', y if $0 < '7';");
- buildPlan("b = foreach x generate (integer)$0;");
+ buildPlan("b = foreach x generate (int)$0;");
buildPlan("c = foreach y generate (bag)$1;");
- buildPlan("d = foreach y generate (integer)($1/2);");
+ buildPlan("d = foreach y generate (int)($1/2);");
}
@Test
@@ -865,6 +865,21 @@
buildPlan("c = foreach y generate $0, ($0 matches 'yuri.*' ? $1 - 10 : $1);");
}
+ @Test
+ public void testQuery74() {
+ buildPlan("a = load 'a' as (field1: int, field2: long);");
+ buildPlan("b = load 'a' as (field1: bytearray, field2: double);");
+ buildPlan("c = group a by field1, b by field1;");
+ buildPlan("d = cogroup a by ((field1+field2)*field1), b by field1;");
+ }
+
+ @Test
+ public void testQuery75() {
+ buildPlan("union (load 'a'), (load 'b'), (load 'c');");
+ }
+
+ // Helper Functions
+
// Helper Functions
// =================
public LogicalPlan buildPlan(String query) {
@@ -914,6 +929,7 @@
}
}
*/
+
assertTrue(lp != null);
return lp;
} catch (IOException e) {
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java Tue May 20 14:41:07 2008
@@ -1428,18 +1428,15 @@
// split
LOSplit split1 = new LOSplit(plan,
genNewOperatorKey(),
- new ArrayList<LogicalOperator>(),
- new HashMap<String,LogicalPlan>());
+ new ArrayList<LogicalOperator>());
// output1
- LOSplitOutput splitOutput1 = new LOSplitOutput(plan, genNewOperatorKey(), 0) ;
+ LOSplitOutput splitOutput1 = new LOSplitOutput(plan, genNewOperatorKey(), 0, innerPlan1) ;
split1.addOutput(splitOutput1);
- split1.addOutputAlias("output1", innerPlan1);
// output2
- LOSplitOutput splitOutput2 = new LOSplitOutput(plan, genNewOperatorKey(), 1) ;
+ LOSplitOutput splitOutput2 = new LOSplitOutput(plan, genNewOperatorKey(), 1, innerPlan2) ;
split1.addOutput(splitOutput2);
- split1.addOutputAlias("output2", innerPlan2);
plan.add(load1);
plan.add(split1);
@@ -1557,18 +1554,15 @@
// split
LOSplit split1 = new LOSplit(plan,
genNewOperatorKey(),
- new ArrayList<LogicalOperator>(),
- new HashMap<String,LogicalPlan>());
+ new ArrayList<LogicalOperator>());
// output1
- LOSplitOutput splitOutput1 = new LOSplitOutput(plan, genNewOperatorKey(), 0) ;
+ LOSplitOutput splitOutput1 = new LOSplitOutput(plan, genNewOperatorKey(), 0, innerPlan1) ;
split1.addOutput(splitOutput1);
- split1.addOutputAlias("output1", innerPlan1);
// output2
- LOSplitOutput splitOutput2 = new LOSplitOutput(plan, genNewOperatorKey(), 1) ;
+ LOSplitOutput splitOutput2 = new LOSplitOutput(plan, genNewOperatorKey(), 1, innerPlan2) ;
split1.addOutput(splitOutput2);
- split1.addOutputAlias("output2", innerPlan2);
plan.add(load1);
plan.add(split1);