You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/06/03 19:21:59 UTC
svn commit: r662844 [1/2] - in /incubator/pig/branches/types: ./
src/org/apache/pig/impl/logicalLayer/
src/org/apache/pig/impl/logicalLayer/schema/ src/org/apache/pig/impl/plan/
test/org/apache/pig/test/ test/org/apache/pig/test/data/DotFiles/ test/org...
Author: gates
Date: Tue Jun 3 10:21:59 2008
New Revision: 662844
URL: http://svn.apache.org/viewvc?rev=662844&view=rev
Log:
PIG-249 Pi's graph comparison utilities.
Added:
incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot
incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/planComparer/
incubator/pig/branches/types/test/org/apache/pig/test/utils/planComparer/LogicalPlanComparer.java
incubator/pig/branches/types/test/org/apache/pig/test/utils/planComparer/PlanStructuralComparer.java
Modified:
incubator/pig/branches/types/build.xml
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java
incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java
incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java
Modified: incubator/pig/branches/types/build.xml
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Tue Jun 3 10:21:59 2008
@@ -17,10 +17,6 @@
<property name="src.lib.dir" value="${basedir}/lib-src/" />
<property name="src.gen.dir" value="${basedir}/src-gen/" />
- <!-- javacc properties -->
- <property name="src.gen.query.parser.dir" value="${src.gen.dir}/org/apache/pig/impl/logicalLayer/parser" />
- <property name="src.gen.script.parser.dir" value="${src.gen.dir}/org/apache/pig/tools/pigscript/parser" />
-
<property name="javacc.home" value="${basedir}/lib" />
<!-- javac properties -->
@@ -63,6 +59,11 @@
<property name="test.log.dir" value="${basedir}/test/logs"/>
<property name="junit.hadoop.conf" value="${user.home}/pigtest/conf/"/>
+ <!-- javacc properties -->
+ <property name="src.gen.query.parser.dir" value="${src.gen.dir}/org/apache/pig/impl/logicalLayer/parser" />
+ <property name="src.gen.script.parser.dir" value="${src.gen.dir}/org/apache/pig/tools/pigscript/parser" />
+ <property name="src.gen.dot.parser.dir" value="${test.src.dir}/org/apache/pig/test/utils/dotGraph/parser" />
+
<!-- ====================================================== -->
<!-- Stuff needed by all targets -->
<!-- ====================================================== -->
@@ -88,6 +89,7 @@
<mkdir dir="${dist.dir}" />
<mkdir dir="${build.classes}" />
<mkdir dir="${test.build.classes}" />
+ <mkdir dir="${src.gen.dot.parser.dir}" />
</target>
<!-- ================================================================== -->
@@ -96,6 +98,7 @@
<target name="clean">
<delete dir="${src.gen.dir}" />
<delete dir="${build.dir}" />
+ <delete dir="${src.gen.dot.parser.dir}" />
</target>
<!-- ================================================================== -->
@@ -105,6 +108,8 @@
<jjtree target="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt" outputdirectory="${src.gen.query.parser.dir}" javacchome="${javacc.home}" />
<javacc target="${src.gen.query.parser.dir}/QueryParser.jj" outputdirectory="${src.gen.query.parser.dir}" javacchome="${javacc.home}" />
<javacc target="${src.dir}/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj" outputdirectory="${src.gen.script.parser.dir}" javacchome="${javacc.home}" />
+ <jjtree target="${test.src.dir}/org/apache/pig/test/utils/dotGraph/Dot.jjt" outputdirectory="${src.gen.dot.parser.dir}" javacchome="${javacc.home}" />
+ <javacc target="${src.gen.dot.parser.dir}/Dot.jj" outputdirectory="${src.gen.dot.parser.dir}" javacchome="${javacc.home}" />
</target>
<!-- ================================================================== -->
@@ -238,6 +243,7 @@
<!-- Excluced because we don't want to run them -->
<exclude name="**/TypeCheckingTestUtil.java" />
<exclude name="**/TypeGraphPrinter.java" />
+ <exclude name="**/LogicalPlanTester.java" />
<exclude name="**/TestHelper.java" />
<exclude name="**/TestLargeFile.java" />
<exclude name="**/TestOrderBy.java" />
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java Tue Jun 3 10:21:59 2008
@@ -238,4 +238,13 @@
public LogicalPlan getPlan() {
return mPlan ;
}
+
+
+ /***
+ * IMPORTANT:
+ * This method is only used for unit testing purpose.
+ */
+ public void setSchemaComputed(boolean computed) {
+ mIsSchemaComputed = computed ;
+ }
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java Tue Jun 3 10:21:59 2008
@@ -517,6 +517,13 @@
Schema other,
boolean relaxInner,
boolean relaxAlias) {
+
+ // If both of them are null, they are equal
+ if ((schema == null) && (other == null)) {
+ return true ;
+ }
+
+ // otherwise
if (schema == null) {
return false ;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/plan/NodeIdGenerator.java Tue Jun 3 10:21:59 2008
@@ -47,4 +47,8 @@
return nextId;
}
+
+ public static void reset(String scope) {
+ theGenerator.scopeToIdMap.put(scope, 0L) ;
+ }
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java Tue Jun 3 10:21:59 2008
@@ -107,21 +107,19 @@
* Get the map of operator key and associated operators
* @return map of operator key and operators.
*/
-/*
public Map<OperatorKey, E> getKeys() {
return mKeys;
}
-*/
+
/**
* Get the map of operators and associated operator keys
* @return map of operator and operator keys.
*/
-/*
+ /*
public Map<E, OperatorKey> getOps() {
return mOps;
}
-*/
-
+ */
/**
* Insert an operator into the plan. This only inserts it as a node in
* the graph, it does not connect it to any other operators. That should
@@ -343,5 +341,9 @@
return tmpList.size() == 1 ;
}
+ public int size() {
+ return mKeys.size() ;
+ }
+
}
Added: incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestSchemaParser.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+import org.junit.Test;
+import org.apache.pig.impl.logicalLayer.parser.QueryParser;
+import org.apache.pig.impl.logicalLayer.parser.ParseException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.data.DataType;
+
+import java.io.ByteArrayInputStream;
+
+import junit.framework.TestCase;
+
+
+public class TestSchemaParser extends TestCase {
+
+ @Test
+ public void test1() throws ParseException {
+ String test = "fieldA: int, fieldB:double,fieldC:bYteaRRay, fieldD:charARRAY";
+ Schema schema = parseSchema(test) ;
+ assertEquals(schema.size(), 4) ;
+ assertAliases(schema, "fieldA", "fieldB", "fieldC", "fieldD") ;
+ assertTypes(schema, DataType.INTEGER, DataType.DOUBLE,
+ DataType.BYTEARRAY, DataType.CHARARRAY);
+ }
+
+ @Test
+ public void test2() throws ParseException {
+ String test = "fieldA: bag {tuple1:tuple(a:int,b:long,c:float,d:double)}";
+ Schema schema = parseSchema(test) ;
+
+ assertEquals(schema.size(), 1);
+ assertEquals(schema.getField(0).type, DataType.BAG) ;
+ assertEquals(schema.getField(0).alias, "fieldA") ;
+
+ Schema inner = schema.getField(0).schema ;
+ assertEquals(inner.size(), 1) ;
+ assertAliases(inner.getField(0).schema, "a", "b", "c", "d") ;
+ assertTypes(inner.getField(0).schema, DataType.INTEGER, DataType.LONG,
+ DataType.FLOAT, DataType.DOUBLE);
+
+ }
+
+ @Test
+ public void test3() throws ParseException {
+ String test = "tuple1: tuple(a:chararray,b:long),"
+ +" tuple2: tuple(c:int,d:float) ";
+ Schema schema = parseSchema(test) ;
+
+ assertEquals(schema.size(), 2);
+ assertAliases(schema, "tuple1", "tuple2") ;
+ assertTypes(schema, DataType.TUPLE, DataType.TUPLE);
+
+ Schema inner1 = schema.getField(0).schema ;
+ assertEquals(inner1.size(), 2);
+ assertAliases(inner1, "a", "b") ;
+ assertTypes(inner1, DataType.CHARARRAY, DataType.LONG);
+
+ Schema inner2 = schema.getField(1).schema ;
+ assertEquals(inner2.size(), 2);
+ assertAliases(inner2, "c", "d") ;
+ assertTypes(inner2, DataType.INTEGER, DataType.FLOAT) ;
+
+ }
+
+ @Test
+ public void test4() throws ParseException {
+ String test = "garage: bag{tuple1: tuple(num_tools: int)}, links: int";
+ Schema schema = parseSchema(test) ;
+
+ // the schema
+ assertEquals(schema.size(), 2) ;
+ assertAliases(schema, "garage", "links") ;
+ assertTypes(schema, DataType.BAG, DataType.INTEGER);
+ assertEquals(schema.getField(0).type, DataType.BAG) ;
+
+ // inner schema
+ Schema inner = schema.getField(0).schema ;
+ assertEquals(inner.size(), 1) ;
+ assertEquals(inner.getField(0).type, DataType.TUPLE) ;
+ assertEquals(inner.getField(0).alias, "tuple1") ;
+
+ // inner of inner
+ Schema innerInner = inner.getField(0).schema ;
+ assertEquals(innerInner.size(), 1) ;
+ assertEquals(innerInner.getField(0).type, DataType.INTEGER) ;
+ }
+
+ private void assertAliases(Schema schema, String... aliases)
+ throws ParseException {
+ for(int i=0; i < aliases.length;i++) {
+ assertEquals(schema.getField(i).alias, aliases[i]);
+ }
+ }
+
+ private void assertTypes(Schema schema, byte... types)
+ throws ParseException {
+ for(int i=0; i < types.length;i++) {
+ assertEquals(schema.getField(i).type, types[i]);
+ }
+ }
+
+ private Schema parseSchema(String schemaString)
+ throws ParseException {
+ ByteArrayInputStream stream = new ByteArrayInputStream(schemaString.getBytes()) ;
+ QueryParser parser = new QueryParser(stream) ;
+ return parser.TupleSchema() ;
+ }
+}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java Tue Jun 3 10:21:59 2008
@@ -20,30 +20,92 @@
import junit.framework.TestCase;
import org.apache.pig.impl.logicalLayer.*;
-import org.apache.pig.impl.logicalLayer.validators.TypeCheckingValidator;
-import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.plan.CompilationMessageCollector;
-import org.apache.pig.impl.plan.OperatorKey;
-import org.apache.pig.impl.plan.PlanValidationException;
-import org.apache.pig.ExecType;
import org.apache.pig.test.utils.TypeCheckingTestUtil;
+import org.apache.pig.test.utils.LogicalPlanTester;
import org.junit.Test;
-
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-import java.io.IOException;
+import org.junit.Before;
public class TestTypeChecking extends TestCase {
+ final String FILE_BASE_LOCATION = "test/org/apache/pig/test/data/DotFiles/" ;
+
+ LogicalPlanTester planTester = new LogicalPlanTester() ;
+
+ @Before
+ public void setUp() {
+ planTester.reset();
+ }
+
+
+ @Test
+ public void testSimple1() throws Throwable {
+ TypeCheckingTestUtil.printCurrentMethodName() ;
+ planTester.buildPlan("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ LogicalPlan plan = planTester.buildPlan("b = distinct a ;") ;
+ planTester.typeCheckAgainstDotFile(plan, FILE_BASE_LOCATION + "plan1.dot");
+ }
+
+
+ @Test
+ public void testByScript1() throws Throwable {
+ TypeCheckingTestUtil.printCurrentMethodName() ;
+ planTester.typeCheckUsingDotFile(FILE_BASE_LOCATION + "testScript1.dot");
+ }
+
+ @Test
+ public void testByScript2() throws Throwable {
+ TypeCheckingTestUtil.printCurrentMethodName() ;
+ planTester.typeCheckUsingDotFile(FILE_BASE_LOCATION + "testScript2.dot");
+ }
+
+ // Problem with schema parser in QueryParser
+ /*
+ @Test
+ public void testByScript4() throws Throwable {
+ TypeCheckingTestUtil.printCurrentMethodName() ;
+ planTester.typeCheckUsingDotFile(FILE_BASE_LOCATION + "testScript4.dot");
+ }
+ */
+
+ /*
+
+ @Test
+ public void testByScript3() throws Throwable {
+ TypeCheckingTestUtil.printCurrentMethodName() ;
+ planTester.typeCheckUsingDotFile(FILE_BASE_LOCATION + "testScript3.dot");
+ }
+
+ */
+
+ // TODO: Convert all of these to dot files
+
+ /*
@Test
public void testValidation1() throws Throwable {
TypeCheckingTestUtil.printCurrentMethodName() ;
- buildPlan("a = load 'a' as (field1: chararray, field2: tuple(inner1 : bytearray, inner2 : int));");
- LogicalPlan plan = buildPlan("b = group a by field1;");
- validatePlan(plan) ;
+ planTester.buildPlan("a = load 'a' as ("
+ + "group: tuple(field1:int, field2:double) ,"
+ + "a: bag{tuple1:tuple(field1: int,field2: long)},"
+ + "b: bag{tuple1:tuple(field1: bytearray,field2: double)} "
+ + " ) ;") ;
+ LogicalPlan plan = planTester.buildPlan("b = group a by field1;");
+ planTester.typeCheckPlan(plan);
+
+
+ }
+ */
+
+ /*
+ @Test
+ public void testValidation1() throws Throwable {
+ TypeCheckingTestUtil.printCurrentMethodName() ;
+ planTester.buildPlan("a = load 'a' as (field1: chararray, field2: tuple(inner1 : bytearray, inner2 : int));");
+ LogicalPlan plan = planTester.buildPlan("b = group a by field1;");
+ planTester.typeCheckPlan(plan);
}
+ */
+ /*
@Test
public void testValidation2() throws Throwable {
TypeCheckingTestUtil.printCurrentMethodName() ;
@@ -119,7 +181,7 @@
LogicalPlan plan = buildPlan("e = foreach a generate name, details.(age, gpa), field3.(a,b) ;");
validatePlan(plan) ;
}
-
+
@Test
public void testValidation11_2() throws Throwable {
TypeCheckingTestUtil.printCurrentMethodName() ;
@@ -137,6 +199,7 @@
LogicalPlan plan = buildPlan("d = foreach c generate a.(field1, field2), b.(field1, field2) ;");
validatePlan(plan) ;
}
+ */
// I suspect there is something wrong in Schema.reconcile()
/*
@@ -164,6 +227,7 @@
}
*/
+ /*
@Test
public void testQuery20() throws Throwable {
TypeCheckingTestUtil.printCurrentMethodName() ;
@@ -179,62 +243,9 @@
String query = "b = foreach a generate (field1+field2)*(field1-field2) ;";
validatePlan(buildPlan(query));
}
-
- /***************** Helpers ******************/
- public LogicalPlan buildPlan(String query) {
- return buildPlan(query, LogicalPlanBuilder.class.getClassLoader());
- }
+ */
+
- public LogicalPlan buildPlan(String query, ClassLoader cldr) {
- LogicalPlanBuilder.classloader = TestTypeChecking.class.getClassLoader() ;
- PigContext pigContext = new PigContext(ExecType.LOCAL);
- LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
-
- try {
- LogicalPlan lp = builder.parse("Test-Plan-Builder",
- query,
- aliases,
- logicalOpTable,
- aliasOp,
- defineAliases);
-
- List<LogicalOperator> roots = lp.getRoots();
-
- if(roots.size() > 0) {
- if (logicalOpTable.get(roots.get(0)) instanceof LogicalOperator){
- System.out.println(query);
- System.out.println(logicalOpTable.get(roots.get(0)));
- }
- if ((roots.get(0)).getAlias()!=null){
- aliases.put(roots.get(0), lp);
- }
- }
-
- assertTrue(lp != null);
-
- return lp ;
- }
- catch (IOException e) {
- fail("IOException: " + e.getMessage());
- }
- catch (Exception e) {
- fail(e.getClass().getName() + ": " + e.getMessage() + " -- " + query);
- }
- return null;
- }
-
- public void validatePlan(LogicalPlan plan) throws PlanValidationException {
- CompilationMessageCollector collector = new CompilationMessageCollector() ;
- TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
- typeValidator.validate(plan, collector) ;
- TypeCheckingTestUtil.printMessageCollector(collector) ;
- TypeCheckingTestUtil.printTypeGraph(plan) ;
- }
-
- Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
- Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
- Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
- Map<String, ExpressionOperator> defineAliases = new HashMap<String, ExpressionOperator>();
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java?rev=662844&r1=662843&r2=662844&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java Tue Jun 3 10:21:59 2008
@@ -19,9 +19,12 @@
import java.util.Iterator;
import java.util.List;
+import java.util.ArrayList;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.*;
+import org.apache.pig.impl.logicalLayer.parser.ParseException ;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.* ;
/**
@@ -31,11 +34,18 @@
public class TypeGraphPrinter extends LOVisitor {
private StringBuilder sb = null ;
+ private int currentTabCount = 0 ;
public TypeGraphPrinter(LogicalPlan plan) {
super(plan, new DependencyOrderWalker<LogicalOperator, LogicalPlan>(plan));
sb = new StringBuilder() ;
}
+
+ private void printTabs() {
+ for (int i=0; i< currentTabCount; i++) {
+ sb.append("\t") ;
+ }
+ }
protected void visit(LogicalOperator op) {
appendOp(op) ;
@@ -53,12 +63,42 @@
appendOp(op) ;
}
- protected void visit(LOCogroup op) {
+ protected void visit(LOCogroup op) throws VisitorException {
appendOp(op) ;
+ List<LogicalOperator> inputs = op.getInputs() ;
+ if (inputs != null) {
+ for(LogicalOperator input: inputs) {
+ List<LogicalPlan> plans
+ = new ArrayList<LogicalPlan>(op.getGroupByPlans().get(input)) ;
+ if (plans != null) {
+ for(LogicalPlan plan: plans) {
+ currentTabCount++ ;
+ printTabs() ;
+ sb.append("<COGroup Inner Plan>\n") ;
+ pushWalker(mCurrentWalker.spawnChildWalker(plan)) ;
+ visit();
+ popWalker();
+ currentTabCount-- ;
+ }
+ }
+ }
+ }
}
- protected void visit(LOGenerate op) {
+ protected void visit(LOGenerate op) throws VisitorException {
appendOp(op) ;
+ List<LogicalPlan> plans = op.getGeneratePlans() ;
+ if (plans != null) {
+ for (LogicalPlan plan: plans) {
+ currentTabCount++ ;
+ printTabs() ;
+ sb.append("<Generate Inner Plan>\n") ;
+ pushWalker(mCurrentWalker.spawnChildWalker(plan)) ;
+ visit();
+ popWalker();
+ currentTabCount-- ;
+ }
+ }
}
protected void visit(LOSort op) {
@@ -67,19 +107,32 @@
protected void visit(LOFilter op) throws VisitorException {
appendOp(op) ;
- sb.append("Filter Inner Plan:\n") ;
- pushWalker(mCurrentWalker.spawnChildWalker(op.getComparisonPlan())) ;
- visit();
- popWalker();
- sb.append("\n") ;
+ if (op.getComparisonPlan() != null) {
+ currentTabCount++ ;
+ printTabs() ;
+ sb.append("<Filter Inner Plan>\n") ;
+ pushWalker(mCurrentWalker.spawnChildWalker(op.getComparisonPlan())) ;
+ visit();
+ popWalker();
+ currentTabCount-- ;
+ }
}
protected void visit(LOSplit op) {
appendOp(op) ;
}
- protected void visit(LOForEach op) {
+ protected void visit(LOForEach op) throws VisitorException {
appendOp(op) ;
+ if (op.getForEachPlan() != null) {
+ currentTabCount++ ;
+ printTabs() ;
+ sb.append("<ForEach Inner Plan>\n") ;
+ pushWalker(mCurrentWalker.spawnChildWalker(op.getForEachPlan())) ;
+ visit();
+ popWalker();
+ currentTabCount-- ;
+ }
}
protected void visit(LOUserFunc op) {
@@ -89,6 +142,14 @@
protected void visit(LOBinCond op) {
appendOp(op) ;
}
+
+ protected void visit(LOCross op) {
+ appendOp(op) ;
+ }
+
+ protected void visit(LOUnion op) {
+ appendOp(op) ;
+ }
protected void visit(LOCast op) {
appendOp(op) ;
@@ -119,15 +180,42 @@
}
- private void appendOp(LogicalOperator op) {
+ private void appendOp(LogicalOperator op) {
+ printTabs() ;
sb.append("(") ;
sb.append(op.getOperatorKey().getId()) ;
sb.append(":") ;
sb.append(op.getClass().getSimpleName()) ;
sb.append("=") ;
- sb.append(DataType.findTypeName(op.getType())) ;
- sb.append("{") ;
- List<LogicalOperator> list = mPlan.getSuccessors(op) ;
+ Schema schema = null ;
+
+ if (!DataType.isComplex(op.getType())) {
+ sb.append(DataType.findTypeName(op.getType())) ;
+ }
+ else {
+ try {
+ if (op instanceof ExpressionOperator) {
+ ExpressionOperator eOp = (ExpressionOperator) op ;
+ Schema.FieldSchema fs = eOp.getFieldSchema() ;
+ Schema.stringifySchema(sb, fs.schema, DataType.TUPLE) ;
+ }
+ else {
+ schema = op.getSchema() ;
+ Schema.stringifySchema(sb, schema, op.getType()) ;
+ }
+ }
+ catch (FrontendException fe) {
+ throw new RuntimeException("PROBLEM PRINTING SCHEMA") ;
+ }
+ catch (ParseException pe) {
+ throw new RuntimeException("PROBLEM PRINTING SCHEMA") ;
+ }
+ }
+
+ sb.append("==>") ;
+ List<LogicalOperator> list
+ = mCurrentWalker.getPlan().getSuccessors(op) ;
+
if (list!=null) {
boolean isFirst = true ;
for(LogicalOperator tmp: list) {
@@ -140,7 +228,9 @@
sb.append(tmp.getOperatorKey().getId()) ;
}
}
- sb.append("}") ;
+ else {
+ sb.append("TERMINAL") ;
+ }
sb.append(")") ;
sb.append("\n") ;
}
Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/plan1.dot Tue Jun 3 10:21:59 2008
@@ -0,0 +1,10 @@
+digraph graph1 {
+ graph [pigScript=" A = LOAD 'tmp' ; B = DISTINCT A ; "] ;
+
+ node [schema="field1: int, field2: float, field3: chararray"] ;
+
+ load [key="10", type="LOLoad"] ;
+ distinct [key="12", type="LODistinct"] ;
+
+ load -> distinct ;
+}
\ No newline at end of file
Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript1.dot Tue Jun 3 10:21:59 2008
@@ -0,0 +1,10 @@
+digraph graph1 {
+ graph [pigScript=" A = LOAD 'tmp' as (field1: int, field2: float, field3: chararray ) ; B = DISTINCT A ; "] ;
+
+ node [schema="field1: int, field2: float, field3: chararray"] ;
+
+ load [key="10", type="LOLoad"] ;
+ distinct [key="20", type="LODistinct"] ;
+
+ load -> distinct ;
+}
\ No newline at end of file
Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript2.dot Tue Jun 3 10:21:59 2008
@@ -0,0 +1,8 @@
+digraph graph1 {
+ graph [pigScript=" A = LOAD 'tmp' ; B = DISTINCT A ; "] ;
+
+ load [key="10", type="LOLoad"] ;
+ distinct [key="20", type="LODistinct"] ;
+
+ load -> distinct ;
+}
\ No newline at end of file
Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript3.dot Tue Jun 3 10:21:59 2008
@@ -0,0 +1,9 @@
+digraph graph1 {
+ graph [pigScript=" a = load 'a' as (field1: long, field2: tuple(inner1 : bytearray, inner2 : float) ) ;
+ b = group a by field2; "] ;
+
+ load [key="10", type="LOLoad", schema="field1: long, field2: tuple(inner1 : bytearray, inner2 : float)"] ;
+ group [key="20", type="LOCogroup" , schema="group: chararray,a: bag{ tuple1:tuple(field1: long,field2: tuple(inner1: bytearray,inner2: integer))}"] ;
+
+ load -> group ;
+}
\ No newline at end of file
Added: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/testScript4.dot Tue Jun 3 10:21:59 2008
@@ -0,0 +1,14 @@
+digraph graph1 {
+ graph [pigScript=" a = load 'a' as (field1: int, field2: long);
+ b = load 'a' as (field1: bytearray, field2: double);
+ c = group a by (field1,field2) , b by (field1,field2) ; "] ;
+
+ load1 [key="10", type="LOLoad", schema="field1: int, field2: long"] ;
+ load2 [key="15", type="LOLoad", schema="field1: bytearray, field2: double"] ;
+ group [key="20", type="LOCogroup" , schema="group: tuple(field1:int, field2:double) ,
+ a: bag{tuple1:tuple(field1: int,field2: long)},
+ b: bag{tuple1:tuple(field1: bytearray,field2: double)} "] ;
+
+ load1 -> group ;
+ load2 -> group ;
+}
\ No newline at end of file
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/LogicalPlanTester.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils;
+
+import org.apache.pig.impl.logicalLayer.*;
+import org.apache.pig.impl.logicalLayer.validators.TypeCheckingValidator;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.PlanValidationException;
+import org.apache.pig.impl.plan.CompilationMessageCollector;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.plan.NodeIdGenerator;
+import org.apache.pig.ExecType;
+import static org.apache.pig.test.utils.TypeCheckingTestUtil.* ;
+import org.apache.pig.test.utils.dotGraph.LogicalPlanLoader;
+import org.apache.pig.test.utils.planComparer.LogicalPlanComparer;
+import org.apache.pig.test.utils.dotGraph.DotGraphReader;
+import org.apache.pig.test.utils.dotGraph.DotGraph;
+
+import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+import java.io.IOException;
+
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
+
+/***
+ * This class is used for logical plan testing
+ */
+public class LogicalPlanTester {
+
+ static final String SCOPE = "scope" ;
+
+ private Map<LogicalOperator, LogicalPlan> aliases = null ;
+ private Map<OperatorKey, LogicalOperator> logicalOpTable = null ;
+ private Map<String, LogicalOperator> aliasOp = null ;
+ private Map<String, ExpressionOperator> defineAliases = null;
+
+ public LogicalPlanTester() {
+ reset() ;
+ }
+
+ /***
+ * Reset state
+ */
+ public void reset() {
+ aliases = new HashMap<LogicalOperator, LogicalPlan>();
+ logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
+ aliasOp = new HashMap<String, LogicalOperator>();
+ defineAliases = new HashMap<String, ExpressionOperator>();
+ NodeIdGenerator.reset(SCOPE);
+ }
+
+ /***
+ * Build plan by the given query string (Pig script)
+ * @param query
+ * @return
+ */
+ public LogicalPlan buildPlan(String query) {
+ return buildPlan(query, LogicalPlanBuilder.class.getClassLoader());
+ }
+
+
+ /***
+ * Type check the given plan
+ * @param plan
+ * @throws PlanValidationException
+ */
+ public void typeCheckPlan(LogicalPlan plan) throws PlanValidationException {
+ CompilationMessageCollector collector = new CompilationMessageCollector() ;
+ TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
+ typeValidator.validate(plan, collector) ;
+ printMessageCollector(collector) ;
+ System.out.println("Actual plan:") ;
+ printTypeGraph(plan) ;
+ }
+
+ /***
+ * Run type checking and compare the result with plan structure
+ * stored in Dot file
+ * @param plan
+ * @param file
+ * @throws PlanValidationException
+ */
+ public void typeCheckAgainstDotFile(LogicalPlan plan, String file)
+ throws PlanValidationException {
+ // validate the given plan
+ typeCheckPlan(plan);
+
+ // load the expected plan from file
+ LogicalPlanLoader planLoader = new LogicalPlanLoader() ;
+ LogicalPlan expectedPlan = planLoader.loadFromFile(file, LogicalPlan.class) ;
+ System.out.println("Expected plan:") ;
+ printTypeGraph(expectedPlan) ;
+
+ // do the comparison
+ LogicalPlanComparer comparer = new LogicalPlanComparer() ;
+ StringBuilder errMsg = new StringBuilder() ;
+ boolean result = comparer.structurallyEquals(plan, expectedPlan, errMsg) ;
+
+ // check
+ System.out.println(errMsg.toString()) ;
+ assertTrue("The expected plan is different", result);
+ System.out.println("Checking DONE!") ;
+ }
+
+ public void typeCheckUsingDotFile(String file)
+ throws PlanValidationException {
+ DotGraphReader reader = new DotGraphReader() ;
+ DotGraph graph = reader.loadFromFile(file) ;
+ if (!graph.attributes.containsKey("pigScript")) {
+ throw new AssertionError("pigScript attribute doesn't exist"
+ + " in Dot file") ;
+ }
+
+ String script = graph.attributes.get("pigScript") ;
+ // TODO: Script splitting here is a quick hack.
+ String[] queries = script.split(";") ;
+ LogicalPlan plan = null ;
+ for(String query : queries) {
+ if (!query.trim().equals("")) {
+ plan = buildPlan(query + ";") ;
+ }
+ }
+ typeCheckAgainstDotFile(plan, file) ;
+
+ }
+
+ ////////////// Helpers ////////////////
+
+ // The actual plan builder
+ private LogicalPlan buildPlan(String query, ClassLoader cldr) {
+
+ LogicalPlanBuilder.classloader = LogicalPlanTester.class.getClassLoader() ;
+ PigContext pigContext = new PigContext(ExecType.LOCAL);
+ LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
+
+ try {
+ LogicalPlan lp = builder.parse(SCOPE,
+ query,
+ aliases,
+ logicalOpTable,
+ aliasOp,
+ defineAliases);
+
+ List<LogicalOperator> roots = lp.getRoots();
+
+ if(roots.size() > 0) {
+ if (logicalOpTable.get(roots.get(0)) instanceof LogicalOperator){
+ System.out.println(query);
+ System.out.println(logicalOpTable.get(roots.get(0)));
+ }
+ if ((roots.get(0)).getAlias()!=null){
+ aliases.put(roots.get(0), lp);
+ }
+ }
+
+ assertTrue(lp != null);
+
+ return lp ;
+ }
+ catch (IOException e) {
+ fail("IOException: " + e.getMessage());
+ }
+ catch (Exception e) {
+ fail(e.getClass().getName() + ": " + e.getMessage() + " -- " + query);
+ }
+ return null;
+ }
+
+
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/Dot.jjt Tue Jun 3 10:21:59 2008
@@ -0,0 +1,203 @@
+options {
+ // Generate non-static functions
+ STATIC = false;
+ // Case is ignored in keywords
+ IGNORE_CASE = true;
+}
+
+PARSER_BEGIN(DOTParser)
+
+package org.apache.pig.test.utils.dotGraph.parser ;
+
+import java.util.*;
+import java.io.*;
+import org.apache.pig.test.utils.dotGraph.* ;
+
+public class DOTParser {
+
+ static String unquote(String s) {
+ return s.substring(1, s.length()-1);
+ }
+
+ static class DotState {
+ public Map<String,String> nodeAttributes = new HashMap<String,String>() ;
+ public Map<String,String> edgeAttributes = new HashMap<String,String>() ;
+ }
+
+}
+
+PARSER_END(DOTParser)
+
+// Skip all the new lines, tabs and spaces
+SKIP : { " " | "\r" | "\t" | "\n" }
+
+MORE :
+{
+ "//" : SINGLE_COMMENT
+ |
+ "#" : SINGLE_COMMENT
+ |
+ "/*" : MULTI_COMMENT
+}
+
+<MULTI_COMMENT> SPECIAL_TOKEN :
+{
+ <("\n" | "\r" | "\r\n")>
+ |
+ <"*/"> : DEFAULT
+}
+
+<SINGLE_COMMENT> SPECIAL_TOKEN :
+{
+ < ("\n" | "\r" | "\r\n") > : DEFAULT
+}
+
+<MULTI_COMMENT> MORE :
+{
+ < ~[] >
+}
+
+<SINGLE_COMMENT> MORE :
+{
+ < ~[] >
+}
+
+
+TOKEN:
+{
+
+ <LPAREN : "{">
+ | <RPAREN : "}">
+ | <LSQBRACKET: "[">
+ | <RSQBRACKET: "]">
+ | <EQUAL: "=">
+ | <COMMA: ",">
+ | <SEMICOLON: ";">
+ | <DIRECTED_EDGE: "->">
+ | <EDGE: "edge">
+ | <NODE: "node">
+ | <GRAPH: "graph">
+ | <DIGRAPH : "digraph">
+ | <#LETTER : ["a"-"z", "A"-"Z"] >
+ | <#DIGIT : ["0"-"9"] >
+ | <#SPECIAL_CHAR : "_" | "$" >
+ | <NAME : <LETTER> ( <LETTER> | <DIGIT> | <SPECIAL_CHAR> )* >
+ | <QUOTEDSTRING : "\"" (~["\""])* "\"">
+}
+
+
+DotGraph Parse() :
+{
+ DotGraph dotGraph = null ;
+ DotState dotState = new DotState() ;
+ Token graphName ;
+}
+{
+ (
+ <DIGRAPH>
+ graphName = <NAME> { dotGraph = new DotGraph(graphName.image) ; }
+ <LPAREN>
+
+ ( LOOKAHEAD(2)
+ EdgeStatement(dotGraph, dotState)
+ | NodeStatement(dotGraph, dotState)
+ | AttributeStatement(dotGraph, dotState)
+ )+
+
+ <RPAREN>
+ )
+ { return dotGraph ; }
+}
+
+void AttributeStatement(DotGraph dotGraph, DotState dotState) :
+{
+ Map<String,String> attributes ;
+}
+{
+ (
+ ( <EDGE> attributes = AttributeList() { dotState.edgeAttributes = attributes ; } )
+ | ( <NODE> attributes = AttributeList() { dotState.nodeAttributes = attributes ; } )
+ | ( <GRAPH> attributes = AttributeList() { dotGraph.attributes = attributes ; } )
+ )
+ <SEMICOLON>
+}
+
+void NodeStatement(DotGraph dotGraph, DotState dotState) :
+{
+ Token nodeName ;
+ DotNode node = new DotNode() ;
+ Map<String,String> attributes ;
+}
+{
+ nodeName = <NAME> { node.name = nodeName.image ; }
+ ( attributes = AttributeList() {
+ node.attributes = new HashMap<String,String>() ;
+ if (dotState != null) {
+ node.attributes.putAll(dotState.nodeAttributes) ;
+ }
+ node.attributes.putAll(attributes) ;
+ }
+ )?
+ <SEMICOLON>
+ { dotGraph.nodes.add(node) ; }
+}
+
+void EdgeStatement(DotGraph dotGraph, DotState dotState) :
+{
+ Token nodeName1 ;
+ Token nodeName2 ;
+ String startingNode ;
+ DotNode node = new DotNode() ;
+ Map<String,String> attributes ;
+}
+{
+ nodeName1 = <NAME> { startingNode = nodeName1.image ; }
+ (
+ <DIRECTED_EDGE>
+ nodeName2 = <NAME>
+ {
+ DotEdge edge = new DotEdge() ;
+ edge.fromNode = startingNode ;
+ edge.toNode = nodeName2.image ;
+
+ dotGraph.edges.add(edge) ;
+
+ startingNode = nodeName2.image ;
+ }
+ )+
+ <SEMICOLON>
+
+}
+
+Map<String,String> AttributeList() :
+{
+ Map<String,String> attributes = new HashMap<String,String>() ;
+ String[] keyValuePair ;
+}
+{
+ (
+ <LSQBRACKET>
+ (keyValuePair = Attribute() { attributes.put(keyValuePair[0], keyValuePair[1]) ; } )
+ (
+ <COMMA>
+ (keyValuePair = Attribute() { attributes.put(keyValuePair[0], keyValuePair[1]) ; } )
+ )*
+ <RSQBRACKET>
+ )
+ { return attributes ; }
+}
+
+String[] Attribute() :
+{
+ Token attName ;
+ Token value ;
+ String[] keyValuePair = new String[2] ;
+}
+{
+ (
+ attName = <NAME> { keyValuePair[0] = attName.image ; }
+ <EQUAL>
+ value = <QUOTEDSTRING> { keyValuePair[1] = unquote(value.image) ; }
+ )
+ { return keyValuePair ; }
+}
\ No newline at end of file
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotEdge.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+/**
+ * This represents an edge in DOT format.
+ * An edge in DOT can have attributes but we're not interested
+ */
+public class DotEdge {
+ public String fromNode ;
+ public String toNode ;
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraph.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+
+/***
+ * This represents graph structure in DOT format
+ */
+public class DotGraph {
+
+ public String name;
+ public List<DotEdge> edges = new ArrayList<DotEdge>() ;
+ public List<DotNode> nodes = new ArrayList<DotNode>() ;
+ public Map<String, String> attributes = new HashMap<String,String>() ;
+
+
+ public DotGraph(String name) {
+ this.name = name ;
+ }
+
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphReader.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.test.utils.dotGraph.parser.DOTParser;
+import org.apache.pig.test.utils.dotGraph.parser.ParseException;
+
+import java.io.*;
+
+/***
+ * This class is responsible for loading textual Dot graph
+ * into object representation.
+ */
+public class DotGraphReader {
+
+ /***
+ * Load Dot graph from string
+ *
+ * @param dotContent the Dot content
+ * @return graph
+ */
+
+ public DotGraph load(String dotContent) {
+ ByteArrayInputStream stream
+ = new ByteArrayInputStream(dotContent.getBytes()) ;
+ DOTParser dotParser = new DOTParser(stream) ;
+ DotGraph graph = null ;
+ try {
+ graph = dotParser.Parse() ;
+ }
+ catch (ParseException pe) {
+ System.out.println(pe.getMessage()) ;
+ throw new RuntimeException("Bad Dot file") ;
+ }
+ return graph ;
+ }
+
+ /***
+ * Convenient method for loading Dot graph from text file
+ * @param file the file containing Dot content
+ * @return graph
+ */
+
+ public DotGraph loadFromFile(String file) {
+ StringBuilder sb = new StringBuilder() ;
+ BufferedReader br = null ;
+ try {
+ br = new BufferedReader(new FileReader(file)) ;
+ String str ;
+ while((str=br.readLine())!=null) {
+ sb.append(str) ;
+ sb.append("\n") ;
+ }
+ }
+ catch (FileNotFoundException fnfe) {
+ throw new RuntimeException("file:" + file + " not found!") ;
+ }
+ catch (IOException ioe) {
+ throw new RuntimeException("Error while reading from:" + file) ;
+ }
+
+ return load(sb.toString()) ;
+ }
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotGraphVisitor.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.logicalLayer.*;
+import org.apache.pig.impl.plan.PlanWalker;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.plan.MultiMap;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
+
+import java.util.Iterator;
+
+/***
+ * Not implemented yet
+ */
+public class DotGraphVisitor extends LOVisitor {
+
+ public DotGraphVisitor(LogicalPlan plan,
+ PlanWalker<LogicalOperator, LogicalPlan> walker) {
+ super(plan, walker);
+ }
+
+ /**
+ * @param lOp
+ * the logical operator that has to be visited
+ * @throws org.apache.pig.impl.plan.VisitorException
+ */
+ protected void visit(LogicalOperator lOp)
+ throws VisitorException {
+ //
+ // Do Nothing
+ //
+ }
+
+ /**
+ * @param eOp
+ * the logical expression operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(ExpressionOperator eOp)
+ throws VisitorException {
+ //
+ // Do Nothing
+ //
+ }
+
+ /**
+ * @param binOp
+ * the logical binary expression operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(BinaryExpressionOperator binOp)
+ throws VisitorException {
+ //
+ // Visit the left hand side operand followed by the right hand side
+ // operand
+ //
+
+ binOp.getLhsOperand().visit(this);
+ binOp.getRhsOperand().visit(this);
+ }
+
+ /**
+ *
+ * @param uniOp
+ * the logical unary operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(UnaryExpressionOperator uniOp) throws VisitorException {
+ // Visit the operand
+
+ uniOp.getOperand().visit(this);
+ }
+
+ /**
+ *
+ * @param cg
+ * the logical cogroup operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(LOCogroup cg) throws VisitorException {
+ // Visit each of the inputs of cogroup.
+ MultiMap<LogicalOperator, LogicalPlan> mapGByPlans = cg.getGroupByPlans();
+ for(LogicalOperator op: cg.getInputs()) {
+ for(LogicalPlan lp: mapGByPlans.get(op)) {
+ if (null != lp) {
+ // TODO FIX - How do we know this should be a
+ // DependencyOrderWalker? We should be replicating the
+ // walker the current visitor is using.
+ PlanWalker w = new DependencyOrderWalker(lp);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: lp.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
+ }
+ }
+ }
+
+ /**
+ *
+ * @param g
+ * the logical generate operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(LOGenerate g) throws VisitorException {
+ // Visit each of generates projection elements.
+ for(LogicalPlan lp: g.getGeneratePlans()) {
+ PlanWalker w = new DependencyOrderWalker(lp);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: lp.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
+ }
+
+ /**
+ *
+ * @param s
+ * the logical sort operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(LOSort s) throws VisitorException {
+ // Visit the sort function
+ for(LogicalPlan lp: s.getSortColPlans()) {
+ PlanWalker w = new DependencyOrderWalker(lp);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: lp.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
+ }
+
+ /**
+ *
+ * @param filter
+ * the logical filter operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(LOFilter filter) throws VisitorException {
+ // Visit the condition for the filter followed by the input
+ PlanWalker w = new DependencyOrderWalker(filter.getComparisonPlan());
+ pushWalker(w);
+ for(LogicalOperator logicalOp: filter.getComparisonPlan().getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
+
+ /**
+ *
+ * @param split
+ * the logical split operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(LOSplit split) throws VisitorException {
+ // Visit each of split's conditions
+ for(LogicalOperator logicalOp: split.getOutputs()) {
+ logicalOp.visit(this);
+ }
+ }
+
+ /**
+ *
+ * @param forEach
+ * the logical foreach operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(LOForEach forEach) throws VisitorException {
+ // Visit the operators that are part of the foreach plan
+ LogicalPlan plan = forEach.getForEachPlan();
+ PlanWalker w = new DependencyOrderWalker(plan);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: plan.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
+
+ /**
+ * Iterate over each expression that is part of the function argument list
+ *
+ * @param func
+ * the user defined function
+ * @throws VisitorException
+ */
+ protected void visit(LOUserFunc func) throws VisitorException {
+ // Visit each of the arguments
+ Iterator<ExpressionOperator> i = func.getArguments().iterator();
+ while (i.hasNext()) {
+ i.next().visit(this);
+ }
+ }
+
+ /**
+ * @param binCond
+ * the logical binCond operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(LOBinCond binCond) throws VisitorException {
+ /*
+ * Visit the conditional expression followed by the left hand operator
+ * and the right hand operator respectively
+ */
+
+ binCond.getCond().visit(this);
+ binCond.getLhsOp().visit(this);
+ binCond.getRhsOp().visit(this);
+ }
+
+ protected void visit(LOCast cast) throws VisitorException {
+ // Visit the expression to be cast
+
+ cast.getExpression().visit(this);
+ }
+
+ protected void visit(LORegexp regexp) throws VisitorException {
+ // Visit the operand of the regexp
+ regexp.getOperand().visit(this);
+ }
+
+ protected void visit(LOLoad load) throws VisitorException{
+
+
+ }
+
+ protected void visit(LOStore store) throws VisitorException{
+
+ }
+
+ protected void visit(LOConst store) throws VisitorException{
+
+ }
+
+ protected void visit(LOUnion u) throws VisitorException {
+
+ }
+
+ protected void visit(LOSplitOutput sop) throws VisitorException {
+ LogicalPlan lp = sop.getConditionPlan();
+ if (null != lp) {
+ PlanWalker w = new DependencyOrderWalker(lp);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: lp.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
+ }
+
+ protected void visit(LODistinct dt) throws VisitorException {
+
+ }
+
+ protected void visit(LOCross cs) throws VisitorException {
+
+ }
+
+ protected void visit(LOProject project) throws VisitorException {
+ // Visit the operand of the project as long as the sentinel is false
+
+ if(!project.getSentinel()) {
+ project.getExpression().visit(this);
+ }
+ }
+
+ public void visit(LOGreaterThan op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOLesserThan op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOGreaterThanEqual op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOLesserThanEqual op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOEqual op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LONotEqual op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOAdd op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOSubtract op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOMultiply op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LODivide op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOMod op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+
+ public void visit(LONegative op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOMapLookup op) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOAnd binOp) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LOOr binOp) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+
+ public void visit(LONot uniOp) throws VisitorException {
+ // TODO Auto-generated method stub
+ return;
+ }
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/DotNode.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import java.util.Map;
+import java.util.HashMap;
+
+/***
+ * This represents a node in DOT format
+ */
+public class DotNode {
+
+ public String name ;
+ public Map<String, String> attributes = new HashMap<String,String>() ;
+
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/ExactKeyMatcher.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.plan.OperatorPlan;
+import org.apache.pig.impl.plan.Operator;
+
+import java.util.Map;
+import java.util.Iterator;
+import java.util.HashMap;
+
+/***
+ * This matcher only does exact key matching
+ */
+public class ExactKeyMatcher implements NodeMatcher<Operator,
+ OperatorPlan<Operator>> {
+
+ public Map<OperatorKey, OperatorKey> match(OperatorPlan<Operator> plan1,
+ OperatorPlan<Operator> plan2,
+ StringBuilder messages) {
+ // Find plan1.OperatorSet - plan2.OperatorSet
+ int diff1 = diffKeys(plan1, plan2, messages, "plan2") ;
+
+ // Find plan2.OperatorSet - plan1.OperatorSet
+ int diff2 = diffKeys(plan2, plan1, messages, "plan1") ;
+
+ // If there is a problem, just finish here
+ if ( (diff1 != 0) || (diff2 != 0) ) {
+ return null ;
+ }
+
+ // if no problem, we just return exact matching
+ Iterator<Operator> iter = plan1.getKeys().values().iterator() ;
+ Map<OperatorKey, OperatorKey> outputMap
+ = new HashMap<OperatorKey, OperatorKey>() ;
+ while(iter.hasNext()) {
+ Operator op = iter.next() ;
+ outputMap.put(op.getOperatorKey(), op.getOperatorKey()) ;
+ }
+
+ return outputMap;
+ }
+
+ /***
+ * Report plan1.OperatorSet - plan2.OperatorSet
+ *
+ * @param plan1
+ * @param plan2
+ * @param messages where the report messages go. null if no messages needed
+ * @param plan2Name the name that is used to refer to plan2 in messages
+ * @return
+ */
+
+ private int diffKeys(OperatorPlan<Operator> plan1,
+ OperatorPlan<Operator> plan2,
+ StringBuilder messages,
+ String plan2Name) {
+ int count = 0 ;
+
+ // prepare
+ Map<OperatorKey, Operator> keyList = plan1.getKeys() ;
+ Iterator<OperatorKey> iter = keyList.keySet().iterator() ;
+
+ // go through the list of vertices of the first plan
+ while(iter.hasNext()) {
+
+ OperatorKey key = iter.next() ;
+
+ // if the same key doesn't exist in the second plan
+ // we've got a problem
+ if (plan2.getOperator(key) == null) {
+ Operator op1 = plan1.getOperator(key) ;
+ if (messages != null) {
+ messages.append(op1.getClass().getSimpleName()) ;
+ appendOpKey(op1.getOperatorKey(), messages) ;
+ messages.append(" doesn't exist") ;
+ if (plan2Name != null) {
+ messages.append(" in ") ;
+ messages.append(plan2Name) ;
+ messages.append("\n") ;
+ }
+ }
+ // increment diff counter
+ count++ ;
+ }
+ }
+
+ return count ;
+ }
+
+ ////////////// String Formatting Helpers //////////////
+
+ protected void appendOpKey(OperatorKey operatorKey, StringBuilder sb) {
+ sb.append("(") ;
+ sb.append(operatorKey.toString()) ;
+ sb.append(")") ;
+ }
+
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/IncreasingKeyMatcher.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.plan.OperatorPlan;
+import org.apache.pig.impl.plan.Operator;
+
+import java.util.*;
+
+/***
+ * This matcher allows matching of different keys but both
+ * key sets have to be in the same increasing order
+ *
+ * Example:
+ * Plan1: 0-Load 1-Distinct 2-Split 3-SplitOutput
+ * 4-SplitOutput 5-Union
+ *
+ * Plan2: 10-Load 14-Distinct 15-Split 100-SplitOutput
+ * 110-SplitOutput 9999-Union
+ *
+ *
+ * Note: All the keys have to be in the same scope.
+ *
+ */
+public class IncreasingKeyMatcher implements
+ NodeMatcher<Operator,OperatorPlan<Operator>> {
+
+ public Map<OperatorKey, OperatorKey> match(OperatorPlan plan1,
+ OperatorPlan plan2,
+ StringBuilder messages) {
+
+ List<OperatorKey> keyList1 = getSortedKeyList(plan1) ;
+ List<OperatorKey> keyList2 = getSortedKeyList(plan2) ;
+
+ // This matching logic only works when both plans have
+ // the same number of operators
+ if (keyList1.size() != keyList2.size()) {
+ messages.append("Two plans have different size") ;
+ return null ;
+ }
+
+ // Populating the output map
+ Map<OperatorKey, OperatorKey> outputMap
+ = new HashMap<OperatorKey, OperatorKey>() ;
+ for(int i=0; i< keyList1.size() ; i++) {
+ outputMap.put(keyList1.get(i), keyList2.get(i)) ;
+ }
+
+ return outputMap ;
+ }
+
+
+ ////////// Helper ///////////
+ private List<OperatorKey> getSortedKeyList(OperatorPlan plan) {
+
+ List<OperatorKey> keyList
+ = new ArrayList<OperatorKey>(plan.getKeys().keySet()) ;
+ Collections.sort(keyList);
+ return keyList ;
+ }
+
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.logicalLayer.*;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.parser.QueryParser ;
+import org.apache.pig.impl.logicalLayer.parser.ParseException ;
+import org.apache.pig.impl.io.FileSpec;
+import org.apache.pig.builtin.PigStorage;
+
+import java.io.IOException;
+import java.io.ByteArrayInputStream;
+import java.util.Map;
+
+
+public class LogicalPlanLoader
+ extends OperatorPlanLoader<LogicalOperator, LogicalPlan> {
+
+ /***
+ * Create various Logical Operators
+ * @param node
+ * @param plan
+ * @return
+ */
+ protected LogicalOperator createOperator(DotNode node, LogicalPlan plan) {
+ String operatorType = node.attributes.get("type") ;
+
+ // Cannot work without the correct type
+ if (operatorType == null) {
+ throw new RuntimeException("Unspecified operator type from Dot file") ;
+ }
+
+ if (operatorType.equals("LOLoad")) {
+ return createLOLoad(node, plan) ;
+ }
+ else if (operatorType.equals("LOFilter")) {
+ return createLOFilter(node, plan) ;
+ }
+ else if (operatorType.equals("LODistinct")) {
+ return createLODistinct(node, plan) ;
+ }
+ else if (operatorType.equals("LOSort")) {
+ return createLOSort(node, plan) ;
+ }
+ else if (operatorType.equals("LOForEach")) {
+ return createLOForEach(node, plan) ;
+ }
+ else if (operatorType.equals("LOSplit")) {
+ return createLOSplit(node, plan) ;
+ }
+ else if (operatorType.equals("LOSplitOutput")) {
+ return createLOSplitOutput(node, plan) ;
+ }
+ else if (operatorType.equals("LOCogroup")) {
+ return createLOCogroup(node, plan) ;
+ }
+ else if (operatorType.equals("LOForEach")) {
+ return createLOForEach(node, plan) ;
+ }
+ else if (operatorType.equals("LOUnion")) {
+ return createLOUnion(node, plan) ;
+ }
+ else if (operatorType.equals("LOCross")) {
+ return createLOCross(node, plan) ;
+ }
+
+ // else
+ throw new AssertionError("Unknown operator type") ;
+ }
+
+ private LOLoad createLOLoad(DotNode node, LogicalPlan plan) {
+ LOLoad load = null ;
+ FileSpec fileSpec = new FileSpec("pi",
+ PigStorage.class.getName()) ;
+ try {
+ load = new LOLoad(plan, getKey(node.attributes), fileSpec, null) ;
+ fillSchema(load, node.attributes) ;
+ }
+ catch (IOException ioe) {
+ throw new AssertionError("Dummy data is not good") ;
+ }
+ return load ;
+ }
+
+ private LOFilter createLOFilter(DotNode node, LogicalPlan plan) {
+ LOFilter filter = new LOFilter(plan, getKey(node.attributes), null, null) ;
+ fillSchema(filter, node.attributes) ;
+ return filter ;
+ }
+
+ private LODistinct createLODistinct(DotNode node, LogicalPlan plan) {
+ LODistinct distinct = new LODistinct(plan, getKey(node.attributes), null) ;
+ fillSchema(distinct, node.attributes) ;
+ return distinct ;
+ }
+
+ private LOSort createLOSort(DotNode node, LogicalPlan plan) {
+ LOSort sort = new LOSort(plan, getKey(node.attributes),
+ null, null, null, "") ;
+ fillSchema(sort, node.attributes) ;
+ return sort ;
+ }
+
+ private LOForEach createLOForEach(DotNode node, LogicalPlan plan) {
+ LOForEach foreach = new LOForEach(plan, getKey(node.attributes), null) ;
+ fillSchema(foreach, node.attributes) ;
+ return foreach ;
+ }
+
+ private LOSplit createLOSplit(DotNode node, LogicalPlan plan) {
+ LOSplit split = new LOSplit(plan, getKey(node.attributes), null) ;
+ fillSchema(split, node.attributes) ;
+ return split ;
+ }
+
+ private LOSplitOutput createLOSplitOutput(DotNode node, LogicalPlan plan) {
+ LOSplitOutput splitOut = new LOSplitOutput(plan,
+ getKey(node.attributes), 0, null) ;
+ fillSchema(splitOut, node.attributes) ;
+ return splitOut ;
+ }
+
+ private LOCogroup createLOCogroup(DotNode node, LogicalPlan plan) {
+ LOCogroup cogroup = new LOCogroup(plan, getKey(node.attributes),
+ null, null, null) ;
+ fillSchema(cogroup, node.attributes) ;
+ return cogroup ;
+ }
+
+ private LOUnion createLOUnion(DotNode node, LogicalPlan plan) {
+ LOUnion union = new LOUnion(plan, getKey(node.attributes), null) ;
+ fillSchema(union, node.attributes) ;
+ return union ;
+ }
+
+ private LOCross createLOCross(DotNode node, LogicalPlan plan) {
+ LOCross cross = new LOCross(plan, getKey(node.attributes), null) ;
+ fillSchema(cross, node.attributes) ;
+ return cross ;
+ }
+
+ private void fillSchema(LogicalOperator op, Map<String,String> attributes) {
+ String schemaString = attributes.get("schema") ;
+ if (schemaString != null) {
+
+ ByteArrayInputStream stream
+ = new ByteArrayInputStream(schemaString.getBytes()) ;
+ QueryParser queryParser = new QueryParser(stream) ;
+ Schema schema = null ;
+ try {
+ schema = queryParser.TupleSchema() ;
+ op.forceSchema(schema);
+ op.setSchemaComputed(true);
+ }
+ catch (ParseException pe) {
+ System.out.println(pe.getMessage()) ;
+ throw new RuntimeException("Error reading schema string") ;
+ }
+ }
+ else {
+ op.forceSchema(null);
+ }
+ }
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/NodeMatcher.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.plan.OperatorPlan;
+import org.apache.pig.impl.plan.Operator;
+
+import java.util.Map;
+
+/***
+ * This is a common interface for graph vertex mapping logic.
+ * Though I call it Matcher so that people don't get confused
+ * with mapper in MapReduce.
+ */
+
+public interface NodeMatcher<E extends Operator,
+ P extends OperatorPlan<E>> {
+
+ /***
+ * This method does matching between vertices in two
+ * given plans.
+ *
+ * @param plan1
+ * @param plan2
+ * @param messages
+ * @return The output map: plan1Key -> plan2Key
+ */
+ Map<OperatorKey, OperatorKey> match(P plan1,
+ P plan2,
+ StringBuilder messages) ;
+}
Added: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java?rev=662844&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/OperatorPlanLoader.java Tue Jun 3 10:21:59 2008
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils.dotGraph;
+
+import org.apache.pig.impl.plan.*;
+
+import java.io.ByteArrayInputStream;
+import java.util.Map;
+import java.util.HashMap;
+
+public abstract class OperatorPlanLoader<E extends Operator,
+ P extends OperatorPlan<E>> {
+
+ /***
+ * This method is used for loading an operator plan encoded in Dot format
+ * @param dotContent the dot content
+ * @param clazz the plan type to be created
+ * @return
+ */
+ public P load(String dotContent, Class<P> clazz) {
+ DotGraphReader dotReader = new DotGraphReader() ;
+ DotGraph graph = dotReader.load(dotContent) ;
+ return constructPlan(graph, clazz) ;
+ }
+
+ /***
+ * Convenient method for loading directly from file
+ * @param file
+ * @param clazz
+ * @return
+ */
+ public P loadFromFile(String file, Class<P> clazz) {
+ DotGraphReader dotReader = new DotGraphReader() ;
+ DotGraph graph = dotReader.loadFromFile(file) ;
+ return constructPlan(graph, clazz) ;
+ }
+
+ public PlanAndGraphEntry loadFromFileWithGraph(String file,
+ Class<P> clazz) {
+ DotGraphReader dotReader = new DotGraphReader() ;
+ DotGraph graph = dotReader.loadFromFile(file) ;
+ P plan = constructPlan(graph, clazz) ;
+ return new PlanAndGraphEntry(plan, graph) ;
+ }
+
+ public class PlanAndGraphEntry {
+ public PlanAndGraphEntry(P plan, DotGraph dotGraph) {
+ this.plan = plan ;
+ this.dotGraph = dotGraph ;
+ }
+ public P plan ;
+ public DotGraph dotGraph ;
+ }
+
+ /***
+ * This method has be overridden to instantiate the correct vertex type
+ *
+ * @param node
+ * @param plan
+ * @return
+ */
+ protected abstract E createOperator(DotNode node, P plan) ;
+
+ //////////////// Helpers //////////////////
+
+ /***
+ * Construct the plan based on the given Dot graph
+ *
+ * @param graph
+ * @return
+ */
+
+ P constructPlan(DotGraph graph, Class<P> clazz) {
+
+ P plan ;
+ Map<String, E> nameMap = new HashMap<String, E>() ;
+
+ try {
+ plan = clazz.newInstance() ;
+ }
+ catch (IllegalAccessException iae) {
+ throw new AssertionError("Cannot instantiate a plan") ;
+ }
+ catch (InstantiationException ie) {
+ throw new AssertionError("Cannot instantiate a plan") ;
+ }
+
+ for(DotNode node: graph.nodes) {
+ E op = createOperator(node, plan) ;
+ nameMap.put(node.name, op) ;
+ plan.add(op);
+ }
+
+ for(DotEdge edge: graph.edges) {
+ E fromOp = nameMap.get(edge.fromNode) ;
+ E toOp = nameMap.get(edge.toNode) ;
+ try {
+ plan.connect(fromOp, toOp);
+ }
+ catch (PlanException pe) {
+ throw new RuntimeException("Invalid Dot file") ;
+ }
+ }
+
+ return plan ;
+ }
+
+
+ /***
+ * Helper for retrieving operator key from encoded attributes.
+ * By default, it will look for "key" in attributes.
+ * If no key is found, an arbitrary one will be generated.
+ * @param attributes
+ * @return
+ */
+ protected OperatorKey getKey(Map<String,String> attributes) {
+ String key = attributes.get("key") ;
+ if (key != null) {
+ return new OperatorKey("scope", Long.parseLong(key)) ;
+ }
+ else {
+ long newId = NodeIdGenerator.getGenerator().getNextNodeId("scope") ;
+ return new OperatorKey("scope", newId) ;
+ }
+ }
+
+}