You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by xu...@apache.org on 2011/05/03 18:58:21 UTC
svn commit: r1099123 [16/16] - in /pig/branches/branch-0.9: ./
src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/impl/ src/org/apache/pig/impl/log...
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/utils/LogicalPlanTester.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/utils/LogicalPlanTester.java?rev=1099123&r1=1099122&r2=1099123&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/utils/LogicalPlanTester.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/utils/LogicalPlanTester.java Tue May 3 16:58:19 2011
@@ -1,295 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pig.test.utils;
-
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.impl.logicalLayer.*;
-import org.apache.pig.impl.logicalLayer.optimizer.LogicalOptimizer;
-import org.apache.pig.impl.logicalLayer.optimizer.SchemaCalculator;
-import org.apache.pig.impl.logicalLayer.optimizer.SchemaRemover;
-import org.apache.pig.impl.logicalLayer.parser.ParseException;
-import org.apache.pig.impl.logicalLayer.validators.TypeCheckingValidator;
-import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.plan.PlanValidationException;
-import org.apache.pig.impl.plan.CompilationMessageCollector;
-import org.apache.pig.impl.plan.OperatorKey;
-import org.apache.pig.impl.plan.NodeIdGenerator;
-import org.apache.pig.impl.plan.VisitorException;
-import org.apache.pig.impl.plan.optimizer.OptimizerException;
-import org.apache.pig.ExecType;
-import static org.apache.pig.test.utils.TypeCheckingTestUtil.* ;
-import org.apache.pig.test.utils.dotGraph.LogicalPlanLoader;
-import org.apache.pig.test.utils.planComparer.LogicalPlanComparer;
-import org.apache.pig.test.utils.dotGraph.DotGraphReader;
-import org.apache.pig.test.utils.dotGraph.DotGraph;
-
-import java.util.List;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.io.IOException;
-
-import static junit.framework.Assert.assertTrue;
-import static junit.framework.Assert.fail;
-
-/***
- * This class is used for logical plan testing
- */
-public class LogicalPlanTester {
-
- static final String SCOPE = "scope" ;
-
- private Map<LogicalOperator, LogicalPlan> aliases = null ;
- private Map<OperatorKey, LogicalOperator> logicalOpTable = null ;
- private Map<String, LogicalOperator> aliasOp = null ;
- private Map<String, String> fileNameMap = null ;
- private PigContext pigContext;
-
- public LogicalPlanTester() {
- this(new PigContext(ExecType.MAPREDUCE, new Properties()));
- }
-
- public LogicalPlanTester(PigContext pc) {
- pigContext = pc;
- reset() ;
- }
-
- /***
- * Reset state
- */
- public void reset() {
- aliases = new HashMap<LogicalOperator, LogicalPlan>();
- logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
- aliasOp = new HashMap<String, LogicalOperator>();
- fileNameMap = new HashMap<String, String>();
- NodeIdGenerator.reset(SCOPE);
- }
-
- /***
- * Build plan by the given query string (Pig script)
- * @param query
- * @return
- */
- public LogicalPlan buildPlan(String query) {
- return buildPlan(query, LogicalPlanBuilder.class.getClassLoader());
- }
-
- public LogicalPlan buildPlanThrowExceptionOnError(String query) throws Exception {
- return buildPlanThrowExceptionOnError(query, LogicalPlanBuilder.class.getClassLoader());
- }
-
-
- /***
- * Type check the given plan
- * @param plan
- * @throws PlanValidationException
- */
- public void typeCheckPlan(LogicalPlan plan) throws PlanValidationException {
- CompilationMessageCollector collector = new CompilationMessageCollector() ;
- TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
- typeValidator.validate(plan, collector) ;
- printMessageCollector(collector) ;
- System.out.println("Actual plan after type check:") ;
- printTypeGraph(plan) ;
- }
-
- public void optimizePlan(LogicalPlan plan) throws OptimizerException {
- LogicalOptimizer optimizer = new LogicalOptimizer(plan);
- optimizer.optimize();
- System.out.println("Actual plan after after optimization:") ;
- printTypeGraph(plan) ;
- }
-
- /***
- * Run type checking and compare the result with plan structure
- * stored in Dot file
- * @param plan
- * @param file
- * @throws PlanValidationException
- */
- public void typeCheckAgainstDotFile(
- LogicalPlan plan,
- String file) throws PlanValidationException, OptimizerException {
- typeCheckAgainstDotFile(plan, file, false);
- }
-
- /***
- * Run type checking and compare the result with plan structure
- * stored in Dot file
- * @param plan
- * @param file
- * @param optimize if true, the plan will be run through the optimizer
- * @throws PlanValidationException
- */
- public void typeCheckAgainstDotFile(
- LogicalPlan plan,
- String file,
- boolean optimize) throws PlanValidationException,
- OptimizerException {
- // validate the given plan
- typeCheckPlan(plan);
-
- if (optimize) optimizePlan(plan);
-
- // load the expected plan from file
- LogicalPlanLoader planLoader = new LogicalPlanLoader() ;
- LogicalPlan expectedPlan = planLoader.loadFromFile(file, LogicalPlan.class) ;
- System.out.println("Expected plan:") ;
- printTypeGraph(expectedPlan) ;
-
- // do the comparison
- LogicalPlanComparer comparer = new LogicalPlanComparer() ;
- StringBuilder errMsg = new StringBuilder() ;
- boolean result = comparer.structurallyEquals(plan, expectedPlan, errMsg) ;
-
- // check
- System.out.println(errMsg.toString()) ;
- assertTrue("The expected plan is different", result);
- System.out.println("Checking DONE!") ;
- }
-
- public void typeCheckUsingDotFile(
- String file) throws PlanValidationException, OptimizerException {
- typeCheckUsingDotFile(file, false);
- }
-
- public void typeCheckUsingDotFile(
- String file,
- boolean optimize) throws PlanValidationException,
- OptimizerException {
- DotGraphReader reader = new DotGraphReader() ;
- DotGraph graph = reader.loadFromFile(file) ;
- if (!graph.attributes.containsKey("pigScript")) {
- throw new AssertionError("pigScript attribute doesn't exist"
- + " in Dot file") ;
- }
-
- String script = graph.attributes.get("pigScript") ;
- // TODO: Script splitting here is a quick hack.
- String[] queries = script.split(";") ;
- LogicalPlan plan = null ;
- for(String query : queries) {
- if (!query.trim().equals("")) {
- plan = buildPlan(query + ";") ;
- }
- }
- typeCheckAgainstDotFile(plan, file, optimize) ;
-
- }
-
- public void printPlan(LogicalPlan lp, String title) {
- try {
- System.err.println(title);
- LOPrinter lv = new LOPrinter(System.err, lp);
- lv.visit();
- System.err.println();
- } catch (Exception e) {
- }
- }
-
- ////////////// Helpers ////////////////
-
- // The actual plan builder
- private LogicalPlan buildPlan(String query, ClassLoader cldr) {
-
- LogicalPlanBuilder.classloader = LogicalPlanTester.class.getClassLoader() ;
- try {
- pigContext.connect();
- } catch (ExecException e1) {
- fail(e1.getClass().getName() + ": " + e1.getMessage() + " -- " + query);
- }
- LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
-
- try {
- return parse(query, builder);
- } catch (IOException e) {
- fail("IOException: " + e.getMessage());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getClass().getName() + ": " + e.getMessage() + " -- " + query);
- }
- return null;
- }
-
- private LogicalPlan parse(String query, LogicalPlanBuilder builder) throws IOException, ParseException {
- LogicalPlan lp = builder.parse(SCOPE,
- query,
- aliases,
- logicalOpTable,
- aliasOp,
- fileNameMap);
-
- List<LogicalOperator> roots = lp.getRoots();
-
- if(roots.size() > 0) {
- if (logicalOpTable.get(roots.get(0)) instanceof LogicalOperator){
- System.out.println(query);
- System.out.println(logicalOpTable.get(roots.get(0)));
- }
- if ((roots.get(0)).getAlias()!=null){
- aliases.put(roots.get(0), lp);
- }
- }
-
- assertTrue(lp != null);
-
- return lp ;
-
- }
-
- private LogicalPlan buildPlanThrowExceptionOnError (String query, ClassLoader cldr) throws IOException, ParseException {
-
- LogicalPlanBuilder.classloader = LogicalPlanTester.class.getClassLoader() ;
-
- try {
- pigContext.connect();
- } catch (ExecException e1) {
- fail(e1.getClass().getName() + ": " + e1.getMessage() + " -- " + query);
- }
- LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
-
- return parse(query, builder);
- }
-
- public void setPlan(LogicalPlan lp) throws VisitorException {
- PlanSetter ps = new PlanSetter(lp);
- ps.visit();
- }
-
- public void setProjectionMap(LogicalPlan lp) throws VisitorException {
- ProjectionMapCalculator pmc = new ProjectionMapCalculator(lp);
- pmc.visit();
- }
-
- public void rebuildProjectionMap(LogicalPlan lp) throws VisitorException {
- ProjectionMapRemover pmr = new ProjectionMapRemover(lp);
- pmr.visit();
- ProjectionMapCalculator pmc = new ProjectionMapCalculator(lp);
- pmc.visit();
- }
-
- public void rebuildSchema(LogicalPlan lp) throws VisitorException {
- SchemaRemover sr = new SchemaRemover(lp);
- sr.visit();
- SchemaCalculator sc = new SchemaCalculator(lp);
- sc.visit();
- }
-
-}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java?rev=1099123&r1=1099122&r2=1099123&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java Tue May 3 16:58:19 2011
@@ -18,41 +18,21 @@
package org.apache.pig.test.utils;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.Properties;
-import org.apache.hadoop.mapred.lib.FieldSelectionMapReduce;
import org.apache.pig.FuncSpec;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.impl.io.FileSpec;
-import org.apache.pig.impl.logicalLayer.LOLoad;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.CompilationMessageCollector;
import org.apache.pig.impl.plan.NodeIdGenerator;
import org.apache.pig.impl.plan.OperatorKey;
-import org.apache.pig.parser.QueryParser.or_cond_return;
-import org.apache.pig.test.TypeGraphPrinter;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
public class TypeCheckingTestUtil {
- public static LOLoad genDummyLOLoad(LogicalPlan plan) {
- String pigStorage = PigStorage.class.getName() ;
- try {
- LOLoad load = new LOLoad(plan,
- genNewOperatorKey(),
- new FileSpec("pi", new FuncSpec(pigStorage)),
- ConfigurationUtil.toConfiguration(new Properties())) ;
- return load ;
- } catch (IOException e) {
- throw new AssertionError("This cannot happen") ;
- }
- }
-
public static org.apache.pig.newplan.logical.relational.LOLoad
genDummyLOLoadNewLP( org.apache.pig.newplan.logical.relational.LogicalPlan plan) {
String pigStorage = PigStorage.class.getName() ;
@@ -90,8 +70,7 @@ public class TypeCheckingTestUtil {
public static void printTypeGraph(LogicalPlan plan) {
System.out.println("*****Type Graph*******") ;
- TypeGraphPrinter printer = new TypeGraphPrinter(plan) ;
- String rep = printer.printToString() ;
+ String rep = plan.toString() ;
System.out.println(rep) ;
}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java?rev=1099123&r1=1099122&r2=1099123&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java Tue May 3 16:58:19 2011
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pig.test.utils.dotGraph;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.pig.FuncSpec;
-import org.apache.pig.builtin.PigStorage;
-import org.apache.pig.data.DataType;
-import org.apache.pig.impl.io.FileSpec;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.LOCogroup;
-import org.apache.pig.impl.logicalLayer.LOCross;
-import org.apache.pig.impl.logicalLayer.LODistinct;
-import org.apache.pig.impl.logicalLayer.LOFilter;
-import org.apache.pig.impl.logicalLayer.LOForEach;
-import org.apache.pig.impl.logicalLayer.LOLoad;
-import org.apache.pig.impl.logicalLayer.LOSort;
-import org.apache.pig.impl.logicalLayer.LOSplit;
-import org.apache.pig.impl.logicalLayer.LOSplitOutput;
-import org.apache.pig.impl.logicalLayer.LOUnion;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.impl.logicalLayer.parser.ParseException;
-import org.apache.pig.impl.logicalLayer.parser.QueryParser;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-
-public class LogicalPlanLoader
- extends OperatorPlanLoader<LogicalOperator, LogicalPlan> {
-
- /***
- * Create various Logical Operators
- * @param node
- * @param plan
- * @return
- */
- protected LogicalOperator createOperator(DotNode node, LogicalPlan plan) {
- String operatorType = node.attributes.get("type") ;
-
- // Cannot work without the correct type
- if (operatorType == null) {
- throw new RuntimeException("Unspecified operator type from Dot file") ;
- }
-
- if (operatorType.equals("LOLoad")) {
- return createLOLoad(node, plan) ;
- }
- else if (operatorType.equals("LOFilter")) {
- return createLOFilter(node, plan) ;
- }
- else if (operatorType.equals("LODistinct")) {
- return createLODistinct(node, plan) ;
- }
- else if (operatorType.equals("LOSort")) {
- return createLOSort(node, plan) ;
- }
- else if (operatorType.equals("LOForEach")) {
- return createLOForEach(node, plan) ;
- }
- else if (operatorType.equals("LOSplit")) {
- return createLOSplit(node, plan) ;
- }
- else if (operatorType.equals("LOSplitOutput")) {
- return createLOSplitOutput(node, plan) ;
- }
- else if (operatorType.equals("LOCogroup")) {
- return createLOCogroup(node, plan) ;
- }
- else if (operatorType.equals("LOForEach")) {
- return createLOForEach(node, plan) ;
- }
- else if (operatorType.equals("LOUnion")) {
- return createLOUnion(node, plan) ;
- }
- else if (operatorType.equals("LOCross")) {
- return createLOCross(node, plan) ;
- }
-
- // else
- throw new AssertionError("Unknown operator type") ;
- }
-
- private LOLoad createLOLoad(DotNode node, LogicalPlan plan) {
- LOLoad load = null ;
- FileSpec fileSpec = new FileSpec("pi",
- new FuncSpec(PigStorage.class.getName())) ;
- try {
- load = new LOLoad(plan, getKey(node.attributes), fileSpec, null) ;
- fillSchema(load, node.attributes) ;
- }
- catch (IOException ioe) {
- throw new AssertionError("Dummy data is not good") ;
- }
- return load ;
- }
-
- private LOFilter createLOFilter(DotNode node, LogicalPlan plan) {
- LOFilter filter = new LOFilter(plan, getKey(node.attributes), null) ;
- fillSchema(filter, node.attributes) ;
- return filter ;
- }
-
- private LODistinct createLODistinct(DotNode node, LogicalPlan plan) {
- LODistinct distinct = new LODistinct(plan, getKey(node.attributes)) ;
- fillSchema(distinct, node.attributes) ;
- return distinct ;
- }
-
- private LOSort createLOSort(DotNode node, LogicalPlan plan) {
- LOSort sort = new LOSort(plan, getKey(node.attributes),
- null, null, null) ;
- fillSchema(sort, node.attributes) ;
- return sort ;
- }
-
- private LOForEach createLOForEach(DotNode node, LogicalPlan plan) {
- LOForEach foreach = new LOForEach(plan, getKey(node.attributes), null, null) ;
- fillSchema(foreach, node.attributes) ;
- return foreach ;
- }
-
- private LOSplit createLOSplit(DotNode node, LogicalPlan plan) {
- LOSplit split = new LOSplit(plan, getKey(node.attributes), null) ;
- fillSchema(split, node.attributes) ;
- return split ;
- }
-
- private LOSplitOutput createLOSplitOutput(DotNode node, LogicalPlan plan) {
- LOSplitOutput splitOut = new LOSplitOutput(plan,
- getKey(node.attributes), 0, null) ;
- fillSchema(splitOut, node.attributes) ;
- return splitOut ;
- }
-
- private LOCogroup createLOCogroup(DotNode node, LogicalPlan plan) {
- LOCogroup cogroup = new LOCogroup(plan, getKey(node.attributes),
- null, null) ;
- fillSchema(cogroup, node.attributes) ;
- return cogroup ;
- }
-
- private LOUnion createLOUnion(DotNode node, LogicalPlan plan) {
- LOUnion union = new LOUnion(plan, getKey(node.attributes)) ;
- fillSchema(union, node.attributes) ;
- return union ;
- }
-
- private LOCross createLOCross(DotNode node, LogicalPlan plan) {
- LOCross cross = new LOCross(plan, getKey(node.attributes)) ;
- fillSchema(cross, node.attributes) ;
- return cross ;
- }
-
- private void fillSchema(LogicalOperator op, Map<String,String> attributes) {
- String schemaString = attributes.get("schema") ;
-
- if (schemaString != null) {
-
- // Replace [NoAlias] with dummy names before set back to null
- // due to the fact that the parser doesn't allow null alias
- int dummyAliasCounter = 0 ;
- String DUMMY_ALIAS_PREFIX = "MY_DUMMY_ALIAS_" ;
- while (schemaString.indexOf("[NoAlias]") != -1) {
- schemaString = schemaString.replaceFirst("\\[NoAlias\\]",
- DUMMY_ALIAS_PREFIX + dummyAliasCounter++) ;
- }
-
- ByteArrayInputStream stream
- = new ByteArrayInputStream(schemaString.getBytes()) ;
- QueryParser queryParser = new QueryParser(stream) ;
- Schema schema = null ;
- try {
- schema = queryParser.TupleSchema() ;
- Schema.setSchemaDefaultType(schema, DataType.BYTEARRAY);
-
- // set all the [NoAlias] to null
- for(int i=0; i < dummyAliasCounter; i++) {
- replaceAliasByNull(schema, DUMMY_ALIAS_PREFIX + i) ;
- }
-
- op.forceSchema(schema);
- op.setSchemaComputed(true);
- }
- catch (ParseException pe) {
- System.out.println(pe.getMessage()) ;
- throw new RuntimeException("Error reading schema string") ;
- }
- }
- else {
- op.forceSchema(null);
- }
- }
-
- private boolean replaceAliasByNull(Schema schema, String alias) {
- if (schema != null) {
- for(int i=0; i < schema.size(); i++) {
- try {
- if ( (schema.getField(i).alias != null) &&
- (schema.getField(i).alias.equals(alias)) ) {
- schema.getField(i).alias = null ;
- return true ;
- }
- // We only do 1 alias per call so having an else
- // here is reasonable
- else {
- if ( (schema.getField(i).type == DataType.BAG) ||
- (schema.getField(i).type == DataType.TUPLE) ) {
- if (replaceAliasByNull(schema.getField(i).schema, alias)) {
- return true ;
- }
- }
- }
-
- } catch (FrontendException e) {
- throw new AssertionError("Cannot access schema internals") ;
- }
- }
- }
- return false ;
- }
-}