You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/01/24 19:15:37 UTC
svn commit: r1062907 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/PigServer.java
src/org/apache/pig/newplan/logical/relational/LOForEach.java
src/org/apache/pig/newplan/logical/relational/LOGenerate.java
test/org/apache/pig/test/TestPigServer.java
Author: daijy
Date: Mon Jan 24 18:15:36 2011
New Revision: 1062907
URL: http://svn.apache.org/viewvc?rev=1062907&view=rev
Log:
PIG-1786: Move describe/nested describe to new logical plan
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOForEach.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
pig/trunk/test/org/apache/pig/test/TestPigServer.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1062907&r1=1062906&r2=1062907&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Jan 24 18:15:36 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1786: Move describe/nested describe to new logical plan (daijy)
+
PIG-1809: addition of TOMAP function (olgan)
PIG-1749: Update Pig parser so that function arguments can contain newline characters (jghoman via daijy)
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1062907&r1=1062906&r2=1062907&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Mon Jan 24 18:15:36 2011
@@ -103,6 +103,8 @@ import org.apache.pig.impl.util.Properti
import org.apache.pig.impl.util.Utils;
import org.apache.pig.newplan.logical.LogicalPlanMigrationVistor;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.newplan.Operator;
import org.apache.pig.pen.ExampleGenerator;
import org.apache.pig.scripting.ScriptEngine;
@@ -725,21 +727,26 @@ public class PigServer {
*/
public Schema dumpSchema(String alias) throws IOException{
try {
- LogicalPlan lp = getPlanFromAlias(alias, "describe");
- lp = compileLp(alias, false);
- Schema schema = null;
- for(LogicalOperator lo : lp.getLeaves()){
- if(lo.getAlias().equals(alias)){
- schema = lo.getSchema();
+ LogicalPlan oldLp = getPlanFromAlias(alias, "describe");
+ oldLp = compileLp(alias, false);
+ LogicalPlanMigrationVistor visitor = new LogicalPlanMigrationVistor(oldLp);
+ visitor.visit();
+ org.apache.pig.newplan.logical.relational.LogicalPlan lp = visitor.getNewLogicalPlan();
+ LogicalSchema schema = null;
+ for(Operator lo : lp.getSinks()){
+ if(((LogicalRelationalOperator)lo).getAlias().equals(alias)){
+ schema = ((LogicalRelationalOperator)lo).getSchema();
break;
}
}
if (schema != null) {
- System.out.println(alias + ": " + schema.toString());
+ Schema s = org.apache.pig.newplan.logical.Util.translateSchema(schema);
+ System.out.println(alias + ": " + s.toString());
+ return s;
} else {
System.out.println("Schema for " + alias + " unknown.");
+ return null;
}
- return schema;
} catch (FrontendException fee) {
int errCode = 1001;
String msg = "Unable to describe schema for alias " + alias;
@@ -754,12 +761,24 @@ public class PigServer {
* @return Schema of alias dumped
* @throws IOException
*/
- public Schema dumpSchemaNested(String alias, String nestedAlias) throws IOException{
- LogicalPlan lp = getPlanFromAlias(alias, "describe");
- lp = compileLp(alias, false);
- LogicalOperator op = lp.getLeaves().get(0);
- if(op instanceof LOForEach) {
- return ((LOForEach)op).dumpNestedSchema(alias, nestedAlias);
+ public Schema dumpSchemaNested(String alias, String nestedAlias) throws FrontendException{
+ LogicalPlan oldLp = getPlanFromAlias(alias, "describe");
+ oldLp = compileLp(alias, false);
+ LogicalPlanMigrationVistor visitor = new LogicalPlanMigrationVistor(oldLp);
+ visitor.visit();
+ org.apache.pig.newplan.logical.relational.LogicalPlan lp = visitor.getNewLogicalPlan();
+ Operator op = lp.getSinks().get(0);
+ if(op instanceof org.apache.pig.newplan.logical.relational.LOForEach) {
+ LogicalSchema nestedSc = ((org.apache.pig.newplan.logical.relational.LOForEach)op).dumpNestedSchema(alias, nestedAlias);
+ if (nestedSc!=null) {
+ Schema s = org.apache.pig.newplan.logical.Util.translateSchema(nestedSc);
+ System.out.println(alias+ "::" + nestedAlias + ": " + s.toString());
+ return s;
+ }
+ else {
+ System.out.println("Schema for "+ alias+ "::" + nestedAlias + " unknown.");
+ return null;
+ }
}
else {
int errCode = 1001;
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOForEach.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOForEach.java?rev=1062907&r1=1062906&r2=1062907&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOForEach.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOForEach.java Mon Jan 24 18:15:36 2011
@@ -24,7 +24,9 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.PlanVisitor;
+import org.apache.pig.newplan.ReverseDependencyOrderWalker;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
+import org.apache.pig.newplan.logical.optimizer.AllSameRalationalNodesVisitor;
public class LOForEach extends LogicalRelationalOperator {
@@ -89,4 +91,32 @@ public class LOForEach extends LogicalRe
}
return innerLoads;
}
+
+ public LogicalSchema dumpNestedSchema(String alias, String nestedAlias) throws FrontendException {
+ NestedRelationalOperatorFinder opFinder = new NestedRelationalOperatorFinder(innerPlan, nestedAlias);
+ opFinder.visit();
+
+ if (opFinder.getMatchedOperator()!=null) {
+ LogicalSchema nestedSc = opFinder.getMatchedOperator().getSchema();
+ return nestedSc;
+ }
+ return null;
+ }
+
+ private static class NestedRelationalOperatorFinder extends AllSameRalationalNodesVisitor {
+ String aliasOfOperator;
+ LogicalRelationalOperator opFound = null;
+ public NestedRelationalOperatorFinder(LogicalPlan plan, String alias) throws FrontendException {
+ super(plan, new ReverseDependencyOrderWalker(plan));
+ aliasOfOperator = alias;
+ }
+ public LogicalRelationalOperator getMatchedOperator() {
+ return opFound;
+ }
+ @Override
+ public void execute(LogicalRelationalOperator op) throws FrontendException {
+ if (op.getAlias().equals(aliasOfOperator))
+ opFound = op;
+ }
+ }
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java?rev=1062907&r1=1062906&r2=1062907&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java Mon Jan 24 18:15:36 2011
@@ -89,7 +89,7 @@ public class LOGenerate extends LogicalR
fieldSchema = exp.getFieldSchema().deepCopy();
expSchema = new LogicalSchema();
- if (fieldSchema.type != DataType.TUPLE && fieldSchema.type != DataType.BAG) {
+ if ((fieldSchema.type != DataType.TUPLE && fieldSchema.type != DataType.BAG)||!flattenFlags[i]) {
// if type is primitive, just add to schema
if (fieldSchema!=null)
expSchema.addField(fieldSchema);
Modified: pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=1062907&r1=1062906&r2=1062907&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigServer.java Mon Jan 24 18:15:36 2011
@@ -716,5 +716,14 @@ public class TestPigServer extends TestC
propertyFile.delete();
FileLocalizer.setInitialized(false);
}
+
+ @Test
+ public void testDescribeForEachFlatten() throws Throwable {
+ pig.registerQuery("a = load 'a';") ;
+ pig.registerQuery("b = group a by $0;") ;
+ pig.registerQuery("c = foreach b generate flatten(a);") ;
+ Schema s = pig.dumpSchema("c") ;
+ assertTrue(s==null);
+ }
}