You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/05/16 20:14:06 UTC
svn commit: r1595278 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/ src/org/apache/pig/impl/
src/org/apache/pig/newplan/logical/optimizer/
src/org/apache/pig/newplan/logical/relational/ src/org/apache/pig/new...
Author: daijy
Date: Fri May 16 18:14:05 2014
New Revision: 1595278
URL: http://svn.apache.org/r1595278
Log:
PIG-3545: Seperate validation rules from optimizer
Added:
pig/trunk/src/org/apache/pig/newplan/logical/visitor/DanglingNestedNodeRemover.java
pig/trunk/src/org/apache/pig/newplan/logical/visitor/InputOutputFileValidatorVisitor.java
Removed:
pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java
pig/trunk/src/org/apache/pig/newplan/logical/rules/InputOutputFileValidator.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/OverwritableStoreFunc.java
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
pig/trunk/src/org/apache/pig/impl/PigContext.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java
pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java
pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java
pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java
pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java
pig/trunk/test/org/apache/pig/test/TestMultiQueryCompiler.java
pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
pig/trunk/test/org/apache/pig/test/TestNewPlanPushUpFilter.java
pig/trunk/test/org/apache/pig/test/TestPigStats.java
pig/trunk/test/org/apache/pig/test/TestProjectRange.java
pig/trunk/test/org/apache/pig/test/TestStore.java
pig/trunk/test/org/apache/pig/test/Util.java
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Arithmetic.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/BinCond.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Comparison.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Generate.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Split1.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Union.gld
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 16 18:14:05 2014
@@ -32,6 +32,8 @@ PIG-2207: Support custom counters for ag
IMPROVEMENTS
+PIG-3545: Seperate validation rules from optimizer (daijy)
+
PIG-3745: Document auto local mode for pig (aniket486)
PIG-3932: Document ROUND_TO builtin UDF (mrflip via cheolsoo)
Modified: pig/trunk/src/org/apache/pig/OverwritableStoreFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/OverwritableStoreFunc.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/OverwritableStoreFunc.java (original)
+++ pig/trunk/src/org/apache/pig/OverwritableStoreFunc.java Fri May 16 18:14:05 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.hadoop.mapreduce.Job;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputFormat;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
-import org.apache.pig.newplan.logical.rules.InputOutputFileValidator;
/**
* A {@link StoreFunc} should implement this interface to enable overwriting its
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Fri May 16 18:14:05 2014
@@ -63,8 +63,6 @@ import org.apache.pig.impl.io.FileLocali
import org.apache.pig.impl.io.FileLocalizer.FetchFileRet;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.plan.CompilationMessageCollector;
-import org.apache.pig.impl.plan.CompilationMessageCollector.MessageType;
import org.apache.pig.impl.streaming.StreamingCommand;
import org.apache.pig.impl.util.LogUtils;
import org.apache.pig.impl.util.PropertiesUtil;
@@ -78,7 +76,6 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor;
import org.apache.pig.newplan.logical.expression.ScalarExpression;
import org.apache.pig.newplan.logical.optimizer.AllExpressionVisitor;
-import org.apache.pig.newplan.logical.optimizer.DanglingNestedNodeRemover;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOStore;
@@ -86,15 +83,6 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LogicalPlanData;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
-import org.apache.pig.newplan.logical.visitor.CastLineageSetter;
-import org.apache.pig.newplan.logical.visitor.ColumnAliasConversionVisitor;
-import org.apache.pig.newplan.logical.visitor.DuplicateForEachColumnRewriteVisitor;
-import org.apache.pig.newplan.logical.visitor.ImplicitSplitInsertVisitor;
-import org.apache.pig.newplan.logical.visitor.ScalarVariableValidator;
-import org.apache.pig.newplan.logical.visitor.ScalarVisitor;
-import org.apache.pig.newplan.logical.visitor.SchemaAliasVisitor;
-import org.apache.pig.newplan.logical.visitor.TypeCheckingRelVisitor;
-import org.apache.pig.newplan.logical.visitor.UnionOnSchemaSetter;
import org.apache.pig.parser.QueryParserDriver;
import org.apache.pig.parser.QueryParserUtils;
import org.apache.pig.pen.ExampleGenerator;
@@ -107,7 +95,6 @@ import org.apache.pig.tools.pigstats.Pig
import org.apache.pig.tools.pigstats.PigStats.JobGraph;
import org.apache.pig.tools.pigstats.ScriptState;
import org.apache.pig.validator.BlackAndWhitelistFilter;
-import org.apache.pig.validator.BlackAndWhitelistValidator;
import org.apache.pig.validator.PigCommandFilter;
/**
@@ -153,8 +140,6 @@ public class PigServer {
protected final String scope = constructScope();
- private boolean aggregateWarning = true;
-
private boolean validateEachStatement = false;
private boolean skipParseInRegisterForBatch = false;
@@ -222,8 +207,6 @@ public class PigServer {
this.pigContext = context;
currDAG = new Graph(false);
- aggregateWarning = "true".equalsIgnoreCase(pigContext.getProperties().getProperty("aggregate.warning"));
-
jobName = pigContext.getProperties().getProperty(
PigContext.JOB_NAME,
PigContext.JOB_NAME_PREFIX + ":DefaultJobName");
@@ -793,6 +776,7 @@ public class PigServer {
*/
public Schema dumpSchema(String alias) throws IOException {
try {
+ pigContext.inDumpSchema = true;
if ("@".equals(alias)) {
alias = getLastRel();
}
@@ -814,6 +798,8 @@ public class PigServer {
int errCode = 1001;
String msg = "Unable to describe schema for alias " + alias;
throw new FrontendException (msg, errCode, PigException.INPUT, false, null, fee);
+ } finally {
+ pigContext.inDumpSchema = false;
}
}
@@ -827,26 +813,31 @@ public class PigServer {
* @throws IOException
*/
public Schema dumpSchemaNested(String alias, String nestedAlias) throws IOException {
- if ("@".equals(alias)) {
- alias = getLastRel();
- }
- Operator op = getOperatorForAlias( alias );
- if( op instanceof LOForEach ) {
- LogicalSchema nestedSc = ((LOForEach)op).dumpNestedSchema(alias, nestedAlias);
- if (nestedSc!=null) {
- Schema s = org.apache.pig.newplan.logical.Util.translateSchema(nestedSc);
- System.out.println(alias+ "::" + nestedAlias + ": " + s.toString());
- return s;
+ try {
+ pigContext.inDumpSchema = true;
+ if ("@".equals(alias)) {
+ alias = getLastRel();
+ }
+ Operator op = getOperatorForAlias( alias );
+ if( op instanceof LOForEach ) {
+ LogicalSchema nestedSc = ((LOForEach)op).dumpNestedSchema(alias, nestedAlias);
+ if (nestedSc!=null) {
+ Schema s = org.apache.pig.newplan.logical.Util.translateSchema(nestedSc);
+ System.out.println(alias+ "::" + nestedAlias + ": " + s.toString());
+ return s;
+ }
+ else {
+ System.out.println("Schema for "+ alias+ "::" + nestedAlias + " unknown.");
+ return null;
+ }
}
else {
- System.out.println("Schema for "+ alias+ "::" + nestedAlias + " unknown.");
- return null;
+ int errCode = 1001;
+ String msg = "Unable to describe schema for " + alias + "::" + nestedAlias;
+ throw new FrontendException (msg, errCode, PigException.INPUT, false, null);
}
- }
- else {
- int errCode = 1001;
- String msg = "Unable to describe schema for " + alias + "::" + nestedAlias;
- throw new FrontendException (msg, errCode, PigException.INPUT, false, null);
+ } finally {
+ pigContext.inDumpSchema = false;
}
}
@@ -1065,6 +1056,7 @@ public class PigServer {
try {
pigContext.inExplain = true;
buildStorePlan( alias );
+ currDAG.lp.optimize(pigContext);
//Only add root xml node if all plans are being written to same stream.
if (format == "xml" && lps == eps) {
@@ -1355,9 +1347,7 @@ public class PigServer {
FrontendException {
// discover pig features used in this script
ScriptState.get().setScriptFeatures(currDAG.lp);
-
- BlackAndWhitelistValidator validator = new BlackAndWhitelistValidator(getPigContext(), currDAG.lp);
- validator.validate();
+ currDAG.lp.optimize(pigContext);
return launchPlan(currDAG.lp, "job_pigexec_");
}
@@ -1692,7 +1682,7 @@ public class PigServer {
QueryParserDriver parserDriver = new QueryParserDriver( pigContext, scope, fileNameMap );
try {
LogicalPlan plan = parserDriver.parse( query );
- compile( plan );
+ plan.validate(pigContext, scope);
} catch(FrontendException ex) {
scriptCache.remove( scriptCache.size() -1 );
throw ex;
@@ -1751,43 +1741,10 @@ public class PigServer {
}
private void compile() throws IOException {
- compile( lp );
+ lp.validate(pigContext, scope);
currDAG.postProcess();
}
- private void compile(LogicalPlan lp) throws FrontendException {
- DanglingNestedNodeRemover DanglingNestedNodeRemover = new DanglingNestedNodeRemover( lp );
- DanglingNestedNodeRemover.visit();
-
- new ColumnAliasConversionVisitor(lp).visit();
- new SchemaAliasVisitor(lp).visit();
- new ScalarVisitor(lp, pigContext, scope).visit();
-
- // ImplicitSplitInsertVisitor has to be called before
- // DuplicateForEachColumnRewriteVisitor. Detail at pig-1766
- new ImplicitSplitInsertVisitor(lp).visit();
-
- // DuplicateForEachColumnRewriteVisitor should be before
- // TypeCheckingRelVisitor which does resetSchema/getSchema
- // heavily
- new DuplicateForEachColumnRewriteVisitor(lp).visit();
-
- CompilationMessageCollector collector = new CompilationMessageCollector() ;
-
- new TypeCheckingRelVisitor( lp, collector).visit();
- new UnionOnSchemaSetter( lp ).visit();
- new CastLineageSetter(lp, collector).visit();
- new ScalarVariableValidator(lp).visit();
- if(aggregateWarning) {
- CompilationMessageCollector.logMessages(collector, MessageType.Warning, aggregateWarning, log);
- } else {
- for(Enum type: MessageType.values()) {
- CompilationMessageCollector.logAllMessages(collector, log);
- }
- }
-
- }
-
private void postProcess() throws IOException {
// The following code deals with store/load combination of
// intermediate files. In this case we will replace the load
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Fri May 16 18:14:05 2014
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URL;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
@@ -32,7 +31,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.pig.PigConstants;
import org.apache.pig.PigException;
import org.apache.pig.backend.BackendException;
import org.apache.pig.backend.datastorage.DataStorage;
@@ -47,32 +45,19 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
import org.apache.pig.backend.hadoop.streaming.HadoopExecutableManager;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.PigImplConstants;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.PlanException;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.streaming.ExecutableManager;
-import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.Utils;
import org.apache.pig.newplan.Operator;
-import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
-import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
-import org.apache.pig.newplan.logical.optimizer.UidResetter;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
-import org.apache.pig.newplan.logical.rules.InputOutputFileValidator;
-import org.apache.pig.newplan.logical.rules.LogicalRelationalNodeValidator;
-import org.apache.pig.newplan.logical.visitor.SortInfoSetter;
-import org.apache.pig.newplan.logical.visitor.StoreAliasSetter;
-import org.apache.pig.pen.POOptimizeDisabler;
import org.apache.pig.tools.pigstats.PigStats;
-import org.apache.pig.validator.BlackAndWhitelistValidator;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
public abstract class HExecutionEngine implements ExecutionEngine {
@@ -100,7 +85,6 @@ public abstract class HExecutionEngine i
// val: the operator key for the root of the phyisical plan
protected Map<OperatorKey, OperatorKey> logicalToPhysicalKeys;
protected Map<Operator, PhysicalOperator> newLogToPhyMap;
- private LogicalPlan newPreoptimizedPlan;
public HExecutionEngine(PigContext pigContext) {
this.pigContext = pigContext;
@@ -245,82 +229,6 @@ public abstract class HExecutionEngine i
throw new FrontendException(msg, errCode, PigException.BUG);
}
- newPreoptimizedPlan = new LogicalPlan(plan);
-
- if (pigContext.inIllustrator) {
- // disable all PO-specific optimizations
- POOptimizeDisabler pod = new POOptimizeDisabler(plan);
- pod.visit();
- }
-
- UidResetter uidResetter = new UidResetter(plan);
- uidResetter.visit();
-
- SchemaResetter schemaResetter = new SchemaResetter(plan,
- true /* skip duplicate uid check*/);
- schemaResetter.visit();
-
- HashSet<String> disabledOptimizerRules;
- try {
- disabledOptimizerRules = (HashSet<String>) ObjectSerializer
- .deserialize(pigContext.getProperties().getProperty(
- PigImplConstants.PIG_OPTIMIZER_RULES_KEY));
- } catch (IOException ioe) {
- int errCode = 2110;
- String msg = "Unable to deserialize optimizer rules.";
- throw new FrontendException(msg, errCode, PigException.BUG, ioe);
- }
- if (disabledOptimizerRules == null) {
- disabledOptimizerRules = new HashSet<String>();
- }
-
- String pigOptimizerRulesDisabled = this.pigContext.getProperties()
- .getProperty(PigConstants.PIG_OPTIMIZER_RULES_DISABLED_KEY);
- if (pigOptimizerRulesDisabled != null) {
- disabledOptimizerRules.addAll(Lists.newArrayList((Splitter.on(",")
- .split(pigOptimizerRulesDisabled))));
- }
-
- if (pigContext.inIllustrator) {
- disabledOptimizerRules.add("MergeForEach");
- disabledOptimizerRules.add("PartitionFilterOptimizer");
- disabledOptimizerRules.add("LimitOptimizer");
- disabledOptimizerRules.add("SplitFilter");
- disabledOptimizerRules.add("PushUpFilter");
- disabledOptimizerRules.add("MergeFilter");
- disabledOptimizerRules.add("PushDownForEachFlatten");
- disabledOptimizerRules.add("ColumnMapKeyPrune");
- disabledOptimizerRules.add("AddForEach");
- disabledOptimizerRules.add("GroupByConstParallelSetter");
- }
-
- StoreAliasSetter storeAliasSetter = new StoreAliasSetter(plan);
- storeAliasSetter.visit();
-
- // run optimizer
- LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer(plan, 100,
- disabledOptimizerRules);
- optimizer.optimize();
-
- // compute whether output data is sorted or not
- SortInfoSetter sortInfoSetter = new SortInfoSetter(plan);
- sortInfoSetter.visit();
-
- if (!pigContext.inExplain) {
- // Validate input/output file. Currently no validation framework in
- // new logical plan, put this validator here first.
- // We might decide to move it out to a validator framework in future
- LogicalRelationalNodeValidator validator = new InputOutputFileValidator(
- plan, pigContext);
- validator.validate();
-
- // Check for blacklist and whitelist properties and disable
- // commands/operators accordingly. Note if a user does not
- // specify these, Pig will work without any filters or validations
- validator = new BlackAndWhitelistValidator(pigContext, plan);
- validator.validate();
- }
-
// translate new logical plan to physical plan
LogToPhyTranslationVisitor translator = new LogToPhyTranslationVisitor(plan);
@@ -354,11 +262,7 @@ public abstract class HExecutionEngine i
}
return result;
}
-
- public LogicalPlan getNewPlan() {
- return newPreoptimizedPlan;
- }
-
+
public PigStats launchPig(LogicalPlan lp, String grpName, PigContext pc)
throws FrontendException, ExecException {
Modified: pig/trunk/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/PigContext.java Fri May 16 18:14:05 2014
@@ -162,6 +162,9 @@ public class PigContext implements Seria
// existence checks, etc).
public boolean inExplain = false;
+ // Where we are processing a dump schema right now
+ public boolean inDumpSchema = false;
+
// whether we're processing an ILLUSTRATE right now.
public boolean inIllustrator = false;
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java Fri May 16 18:14:05 2014
@@ -18,19 +18,48 @@
package org.apache.pig.newplan.logical.relational;
+import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import org.apache.pig.PigConstants;
+import org.apache.pig.PigException;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.PigImplConstants;
import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.plan.CompilationMessageCollector;
+import org.apache.pig.impl.plan.CompilationMessageCollector.MessageType;
import org.apache.pig.impl.util.HashOutputStream;
+import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.newplan.BaseOperatorPlan;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.logical.DotLOPrinter;
+import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanPrinter;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
+import org.apache.pig.newplan.logical.optimizer.UidResetter;
+import org.apache.pig.newplan.logical.visitor.CastLineageSetter;
+import org.apache.pig.newplan.logical.visitor.ColumnAliasConversionVisitor;
+import org.apache.pig.newplan.logical.visitor.DanglingNestedNodeRemover;
+import org.apache.pig.newplan.logical.visitor.DuplicateForEachColumnRewriteVisitor;
+import org.apache.pig.newplan.logical.visitor.ImplicitSplitInsertVisitor;
+import org.apache.pig.newplan.logical.visitor.InputOutputFileValidatorVisitor;
+import org.apache.pig.newplan.logical.visitor.ScalarVariableValidator;
+import org.apache.pig.newplan.logical.visitor.ScalarVisitor;
+import org.apache.pig.newplan.logical.visitor.SchemaAliasVisitor;
+import org.apache.pig.newplan.logical.visitor.SortInfoSetter;
+import org.apache.pig.newplan.logical.visitor.StoreAliasSetter;
+import org.apache.pig.newplan.logical.visitor.TypeCheckingRelVisitor;
+import org.apache.pig.newplan.logical.visitor.UnionOnSchemaSetter;
+import org.apache.pig.pen.POOptimizeDisabler;
+import org.apache.pig.validator.BlackAndWhitelistValidator;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
@@ -136,4 +165,106 @@ public class LogicalPlan extends BaseOpe
return Integer.toString(hos.getHashCode().asInt());
}
+
+ public void validate(PigContext pigContext, String scope) throws FrontendException {
+
+ new DanglingNestedNodeRemover(this).visit();
+ new ColumnAliasConversionVisitor(this).visit();
+ new SchemaAliasVisitor(this).visit();
+ new ScalarVisitor(this, pigContext, scope).visit();
+
+ // ImplicitSplitInsertVisitor has to be called before
+ // DuplicateForEachColumnRewriteVisitor. Detail at pig-1766
+ new ImplicitSplitInsertVisitor(this).visit();
+
+ // DuplicateForEachColumnRewriteVisitor should be before
+ // TypeCheckingRelVisitor which does resetSchema/getSchema
+ // heavily
+ new DuplicateForEachColumnRewriteVisitor(this).visit();
+
+ CompilationMessageCollector collector = new CompilationMessageCollector() ;
+
+ new TypeCheckingRelVisitor( this, collector).visit();
+
+ boolean aggregateWarning = "true".equalsIgnoreCase(pigContext.getProperties().getProperty("aggregate.warning"));
+
+ if(aggregateWarning) {
+ CompilationMessageCollector.logMessages(collector, MessageType.Warning, aggregateWarning, log);
+ } else {
+ for(Enum type: MessageType.values()) {
+ CompilationMessageCollector.logAllMessages(collector, log);
+ }
+ }
+
+ new UnionOnSchemaSetter(this).visit();
+ new CastLineageSetter(this, collector).visit();
+ new ScalarVariableValidator(this).visit();
+ new StoreAliasSetter(this).visit();
+
+ // compute whether output data is sorted or not
+ new SortInfoSetter(this).visit();
+
+ if (!(pigContext.inExplain || pigContext.inDumpSchema)) {
+ // Validate input/output file
+ new InputOutputFileValidatorVisitor(this, pigContext).visit();
+ }
+
+ BlackAndWhitelistValidator validator = new BlackAndWhitelistValidator(pigContext, this);
+ validator.validate();
+
+ // Now make sure the plan is consistent
+ UidResetter uidResetter = new UidResetter(this);
+ uidResetter.visit();
+
+ SchemaResetter schemaResetter = new SchemaResetter(this,
+ true /* skip duplicate uid check*/);
+ schemaResetter.visit();
+ }
+
+ public void optimize(PigContext pigContext) throws FrontendException {
+ if (pigContext.inIllustrator) {
+ // disable all PO-specific optimizations
+ POOptimizeDisabler pod = new POOptimizeDisabler(this);
+ pod.visit();
+ }
+
+ HashSet<String> disabledOptimizerRules;
+ try {
+ disabledOptimizerRules = (HashSet<String>) ObjectSerializer
+ .deserialize(pigContext.getProperties().getProperty(
+ PigImplConstants.PIG_OPTIMIZER_RULES_KEY));
+ } catch (IOException ioe) {
+ int errCode = 2110;
+ String msg = "Unable to deserialize optimizer rules.";
+ throw new FrontendException(msg, errCode, PigException.BUG, ioe);
+ }
+ if (disabledOptimizerRules == null) {
+ disabledOptimizerRules = new HashSet<String>();
+ }
+
+ String pigOptimizerRulesDisabled = pigContext.getProperties()
+ .getProperty(PigConstants.PIG_OPTIMIZER_RULES_DISABLED_KEY);
+ if (pigOptimizerRulesDisabled != null) {
+ disabledOptimizerRules.addAll(Lists.newArrayList((Splitter.on(",")
+ .split(pigOptimizerRulesDisabled))));
+ }
+
+ if (pigContext.inIllustrator) {
+ disabledOptimizerRules.add("MergeForEach");
+ disabledOptimizerRules.add("PartitionFilterOptimizer");
+ disabledOptimizerRules.add("LimitOptimizer");
+ disabledOptimizerRules.add("SplitFilter");
+ disabledOptimizerRules.add("PushUpFilter");
+ disabledOptimizerRules.add("MergeFilter");
+ disabledOptimizerRules.add("PushDownForEachFlatten");
+ disabledOptimizerRules.add("ColumnMapKeyPrune");
+ disabledOptimizerRules.add("AddForEach");
+ disabledOptimizerRules.add("GroupByConstParallelSetter");
+ }
+
+ // run optimizer
+ LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer(this, 100,
+ disabledOptimizerRules);
+ optimizer.optimize();
+ }
}
Added: pig/trunk/src/org/apache/pig/newplan/logical/visitor/DanglingNestedNodeRemover.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/DanglingNestedNodeRemover.java?rev=1595278&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/DanglingNestedNodeRemover.java (added)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/DanglingNestedNodeRemover.java Fri May 16 18:14:05 2014
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.newplan.logical.visitor;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.newplan.DependencyOrderWalker;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.OperatorPlan;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOGenerate;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
+
+public class DanglingNestedNodeRemover extends LogicalRelationalNodesVisitor {
+
+ public DanglingNestedNodeRemover(OperatorPlan plan)
+ throws FrontendException {
+ super(plan, new DependencyOrderWalker(plan));
+ }
+
+ @Override
+ public void visit(LOForEach foreach) throws FrontendException {
+ LogicalPlan innerPlan = foreach.getInnerPlan();
+ List<Operator> opsToRemove = new ArrayList<Operator>();
+ Iterator<Operator> ops = innerPlan.getOperators();
+
+ while (ops.hasNext()) {
+ Operator op = ops.next();
+ // Check if op leads to LOGenerate, otherwise, candidate to remove
+ Operator currentOp = op;
+ boolean endWithNoLOGenerate = false;
+ while (!(currentOp instanceof LOGenerate)) {
+ if (innerPlan.getSuccessors(currentOp)==null) {
+ endWithNoLOGenerate = true;
+ break;
+ }
+ currentOp = innerPlan.getSuccessors(currentOp).get(0);
+ }
+ if (endWithNoLOGenerate)
+ opsToRemove.add(op);
+ }
+
+ for (Operator op : opsToRemove) {
+ innerPlan.removeAndReconnect(op);
+ }
+ }
+}
Added: pig/trunk/src/org/apache/pig/newplan/logical/visitor/InputOutputFileValidatorVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/InputOutputFileValidatorVisitor.java?rev=1595278&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/InputOutputFileValidatorVisitor.java (added)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/InputOutputFileValidatorVisitor.java Fri May 16 18:14:05 2014
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.newplan.logical.visitor;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.OverwritableStoreFunc;
+import org.apache.pig.PigException;
+import org.apache.pig.ResourceSchema;
+import org.apache.pig.StoreFuncInterface;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.newplan.DepthFirstWalker;
+import org.apache.pig.newplan.OperatorPlan;
+import org.apache.pig.newplan.logical.relational.LOStore;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
+
+public class InputOutputFileValidatorVisitor extends LogicalRelationalNodesVisitor {
+ private PigContext pigCtx;
+ public InputOutputFileValidatorVisitor(OperatorPlan plan, PigContext pigContext) throws FrontendException {
+ super(plan, new DepthFirstWalker(plan));
+ pigCtx = pigContext;
+ }
+
+ @Override
+ public void visit(LOStore store) throws FrontendException {
+ StoreFuncInterface sf = store.getStoreFunc();
+ String outLoc = store.getOutputSpec().getFileName();
+ int errCode = 2116;
+ String validationErrStr ="Output Location Validation Failed for: '" + outLoc ;
+ Job dummyJob;
+
+ try {
+ if(store.getSchema() != null){
+ sf.checkSchema(new ResourceSchema(store.getSchema(), store.getSortInfo()));
+ }
+ dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
+ sf.setStoreLocation(outLoc, dummyJob);
+ } catch (Exception ioe) {
+ if(ioe instanceof PigException){
+ errCode = ((PigException)ioe).getErrorCode();
+ }
+ String exceptionMsg = ioe.getMessage();
+ validationErrStr += (exceptionMsg == null) ? "" : " More info to follow:\n" +exceptionMsg;
+ throw new VisitorException(store, validationErrStr, errCode, pigCtx.getErrorSource(), ioe);
+ }
+
+ validationErrStr += " More info to follow:\n";
+ try {
+ sf.getOutputFormat().checkOutputSpecs(dummyJob);
+ } catch (IOException ioe) {
+ byte errSrc = pigCtx.getErrorSource();
+ switch(errSrc) {
+ case PigException.BUG:
+ errCode = 2002;
+ break;
+ case PigException.REMOTE_ENVIRONMENT:
+ errCode = 6000;
+ break;
+ case PigException.USER_ENVIRONMENT:
+ errCode = 4000;
+ break;
+ }
+
+ boolean shouldThrowException = true;
+ if (sf instanceof OverwritableStoreFunc) {
+ if (((OverwritableStoreFunc) sf).shouldOverwrite()) {
+ if (ioe instanceof FileAlreadyExistsException
+ || ioe instanceof org.apache.hadoop.fs.FileAlreadyExistsException) {
+ shouldThrowException = false;
+ }
+ }
+ }
+ if (shouldThrowException) {
+ validationErrStr += ioe.getMessage();
+ throw new VisitorException(store, validationErrStr,
+ errCode, errSrc, ioe);
+ }
+ } catch (InterruptedException ie) {
+ validationErrStr += ie.getMessage();
+ throw new VisitorException(store, validationErrStr, errCode, pigCtx.getErrorSource(), ie);
+ }
+ }
+}
Modified: pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java (original)
+++ pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java Fri May 16 18:14:05 2014
@@ -230,8 +230,9 @@ public class ExampleGenerator {
}
PhysicalPlan compilePlan(LogicalPlan plan) throws ExecException, FrontendException {
+ newPlan = new LogicalPlan(plan);
+ plan.optimize(pigContext);
PhysicalPlan result = execEngine.compile(plan, null);
- newPlan = execEngine.getNewPlan();
return result;
}
Modified: pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java (original)
+++ pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java Fri May 16 18:14:05 2014
@@ -135,7 +135,7 @@ public class TestErrorHandling {
pig.registerQuery( query );
} catch(FrontendException ex) {
System.out.println( ex.getCause().getMessage() );
- Assert.assertTrue( ex.getCause().getMessage().contains( "line 2, column 0" ) );
+ Assert.assertTrue( ex.getMessage().contains( "line 2, column 0" ) );
Assert.assertTrue( ex.getCause().getMessage().contains( "No FileSystem for scheme: fs2you" ) );
return;
}
Modified: pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java Fri May 16 18:14:05 2014
@@ -47,7 +47,7 @@ import org.apache.pig.impl.util.LogUtils
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOStore;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
-import org.apache.pig.newplan.logical.rules.InputOutputFileValidator;
+import org.apache.pig.newplan.logical.visitor.InputOutputFileValidatorVisitor;
import org.junit.Before;
import org.junit.Test;
@@ -68,8 +68,8 @@ public class TestInputOutputFileValidato
String outputfile = generateNonExistenceTempFile().getAbsolutePath() ;
LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getFs()) ;
- InputOutputFileValidator executor = new InputOutputFileValidator(plan, ctx) ;
- executor.validate() ;
+ InputOutputFileValidatorVisitor visitor = new InputOutputFileValidatorVisitor(plan, ctx) ;
+ visitor.visit();
}
@Test(expected = VisitorException.class) //should expect an exception
@@ -79,8 +79,8 @@ public class TestInputOutputFileValidato
LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;
- InputOutputFileValidator executor = new InputOutputFileValidator(plan, ctx) ;
- executor.validate() ;
+ InputOutputFileValidatorVisitor visitor = new InputOutputFileValidatorVisitor(plan, ctx) ;
+ visitor.visit() ;
}
/**
Modified: pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java Fri May 16 18:14:05 2014
@@ -49,7 +49,7 @@ import org.apache.pig.impl.util.LogUtils
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOStore;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
-import org.apache.pig.newplan.logical.rules.InputOutputFileValidator;
+import org.apache.pig.newplan.logical.visitor.InputOutputFileValidatorVisitor;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
@@ -78,8 +78,8 @@ public class TestInputOutputMiniClusterF
LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;
- InputOutputFileValidator executor = new InputOutputFileValidator(plan, ctx) ;
- executor.validate() ;
+ InputOutputFileValidatorVisitor executor = new InputOutputFileValidatorVisitor(plan, ctx) ;
+ executor.visit() ;
}
@Test
@@ -89,9 +89,9 @@ public class TestInputOutputMiniClusterF
LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;
- InputOutputFileValidator executor = new InputOutputFileValidator(plan, ctx) ;
+ InputOutputFileValidatorVisitor executor = new InputOutputFileValidatorVisitor(plan, ctx) ;
try {
- executor.validate() ;
+ executor.visit() ;
fail("Excepted to fail.");
} catch(Exception e) {
//good
Modified: pig/trunk/test/org/apache/pig/test/TestMultiQueryCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMultiQueryCompiler.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMultiQueryCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMultiQueryCompiler.java Fri May 16 18:14:05 2014
@@ -1524,6 +1524,7 @@ public class TestMultiQueryCompiler {
private PhysicalPlan checkPhysicalPlan(LogicalPlan lp, int expectedRoots,
int expectedLeaves, int expectedSize) throws IOException {
+ lp.optimize(myPig.getPigContext());
System.out.println("===== check physical plan =====");
PhysicalPlan pp = ((MRExecutionEngine)myPig.getPigContext().getExecutionEngine()).compile(
Modified: pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java Fri May 16 18:14:05 2014
@@ -622,6 +622,7 @@ public class TestMultiQueryLocal {
private PhysicalPlan checkPhysicalPlan(LogicalPlan lp, int expectedRoots,
int expectedLeaves, int expectedSize) throws IOException {
+ lp.optimize(myPig.getPigContext());
System.out.println("===== check physical plan =====");
PhysicalPlan pp = ((MRExecutionEngine)myPig.getPigContext().getExecutionEngine()).compile(
Modified: pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java Fri May 16 18:14:05 2014
@@ -17,7 +17,7 @@
*/
package org.apache.pig.test;
-
+import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -49,6 +49,7 @@ import org.apache.pig.newplan.optimizer.
import org.apache.pig.newplan.optimizer.Rule;
import org.apache.pig.test.utils.Identity;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
/**
@@ -57,6 +58,11 @@ import org.junit.Test;
public class TestNewPlanPushDownForeachFlatten {
PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
+ @BeforeClass
+ public static void oneTimeSetup() throws Exception {
+ new File("dummy").delete();
+ }
+
/**
*
* A simple filter UDF for testing
Modified: pig/trunk/test/org/apache/pig/test/TestNewPlanPushUpFilter.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNewPlanPushUpFilter.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestNewPlanPushUpFilter.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestNewPlanPushUpFilter.java Fri May 16 18:14:05 2014
@@ -17,6 +17,7 @@
*/
package org.apache.pig.test;
+import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -52,6 +53,7 @@ import org.apache.pig.newplan.optimizer.
import org.apache.pig.newplan.optimizer.Rule;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
/**
@@ -60,6 +62,11 @@ import org.junit.Test;
public class TestNewPlanPushUpFilter {
PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
+ @BeforeClass
+ public static void oneTimeSetup() throws Exception {
+ new File("dummy").delete();
+ }
+
@Before
public void tearDown() {
}
Modified: pig/trunk/test/org/apache/pig/test/TestPigStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStats.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStats.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStats.java Fri May 16 18:14:05 2014
@@ -166,6 +166,7 @@ public class TestPigStats {
pig.registerQuery("store E into 'alias_output';");
LogicalPlan lp = getLogicalPlan(pig);
+ lp.optimize(pig.getPigContext());
PhysicalPlan pp = ((MRExecutionEngine)pig.getPigContext().getExecutionEngine()).compile(lp,
null);
MROperPlan mp = getMRPlan(pp, pig.getPigContext());
Modified: pig/trunk/test/org/apache/pig/test/TestProjectRange.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestProjectRange.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestProjectRange.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestProjectRange.java Fri May 16 18:14:05 2014
@@ -784,13 +784,7 @@ public class TestProjectRange {
private LogicalPlan createAndProcessLPlan(String query) throws FrontendException {
//TODO: create a common util function for logical plan tests
LogicalPlan lp = generateLogicalPlan(query);
- new ColumnAliasConversionVisitor( lp ).visit();
- new SchemaAliasVisitor( lp ).visit();
-
- CompilationMessageCollector collector = new CompilationMessageCollector() ;
- new TypeCheckingRelVisitor( lp, collector).visit();
- new UnionOnSchemaSetter( lp ).visit();
- new CastLineageSetter(lp, collector).visit();
+ lp.validate(pigServer.getPigContext(), "test");
return lp;
Modified: pig/trunk/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestStore.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestStore.java Fri May 16 18:14:05 2014
@@ -26,6 +26,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -78,7 +79,7 @@ import org.apache.pig.impl.util.Utils;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.relational.LOStore;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
-import org.apache.pig.newplan.logical.rules.InputOutputFileValidator;
+import org.apache.pig.newplan.logical.visitor.InputOutputFileValidatorVisitor;
import org.apache.pig.parser.ParserException;
import org.apache.pig.parser.QueryParserDriver;
import org.apache.pig.test.utils.GenRandomData;
@@ -149,7 +150,6 @@ public class TestStore {
"i:int,d:double);" +
"store a into '" + outputFileName + "' using " + "PigStorage();";
org.apache.pig.newplan.logical.relational.LogicalPlan lp = Util.buildLp( pig, query );
- new InputOutputFileValidator(lp, pig.getPigContext()).validate();
} catch (PlanValidationException e){
// Since output file is not present, validation should pass
// and not throw this exception.
@@ -169,9 +169,9 @@ public class TestStore {
String query = "a = load '" + inputFileName + "' as (c:chararray, " +
"i:int,d:double);" +
"store a into '" + outputFileName + "' using PigStorage();";
- org.apache.pig.newplan.logical.relational.LogicalPlan lp = Util.buildLp( pig, query );
- new InputOutputFileValidator(lp, pig.getPigContext()).validate();
- } catch (FrontendException pve){
+ Util.buildLp( pig, query );
+ } catch (InvocationTargetException e){
+ FrontendException pve = (FrontendException)e.getCause();
// Since output file is present, validation should fail
// and throw this exception
assertEquals(6000,pve.getErrorCode());
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Fri May 16 18:14:05 2014
@@ -86,7 +86,6 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
import org.apache.pig.impl.plan.CompilationMessageCollector;
import org.apache.pig.impl.util.LogUtils;
-import org.apache.pig.newplan.logical.optimizer.DanglingNestedNodeRemover;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanPrinter;
import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
import org.apache.pig.newplan.logical.optimizer.UidResetter;
@@ -96,6 +95,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
import org.apache.pig.newplan.logical.visitor.CastLineageSetter;
import org.apache.pig.newplan.logical.visitor.ColumnAliasConversionVisitor;
+import org.apache.pig.newplan.logical.visitor.DanglingNestedNodeRemover;
import org.apache.pig.newplan.logical.visitor.ScalarVisitor;
import org.apache.pig.newplan.logical.visitor.SchemaAliasVisitor;
import org.apache.pig.newplan.logical.visitor.SortInfoSetter;
@@ -1058,6 +1058,7 @@ public class Util {
public static PhysicalPlan buildPp(PigServer pigServer, String query)
throws Exception {
LogicalPlan lp = buildLp( pigServer, query );
+ lp.optimize(pigServer.getPigContext());
return ((MRExecutionEngine)pigServer.getPigContext().getExecutionEngine()).compile(lp,
pigServer.getPigContext().getProperties());
}
@@ -1067,16 +1068,7 @@ public class Util {
QueryParserDriver parserDriver = new QueryParserDriver( pc, "test", fileNameMap );
org.apache.pig.newplan.logical.relational.LogicalPlan lp = parserDriver.parse( query );
- new ColumnAliasConversionVisitor(lp).visit();
- new SchemaAliasVisitor(lp).visit();
- new ScalarVisitor(lp, pc, "test").visit();
-
- CompilationMessageCollector collector = new CompilationMessageCollector() ;
-
- new TypeCheckingRelVisitor( lp, collector).visit();
-
- new UnionOnSchemaSetter( lp ).visit();
- new CastLineageSetter(lp, collector).visit();
+ lp.validate(pc, "test");
return lp;
}
@@ -1085,16 +1077,7 @@ public class Util {
QueryParserDriver parserDriver = new QueryParserDriver( pc, "test", fileNameMap );
org.apache.pig.newplan.logical.relational.LogicalPlan lp = parserDriver.parse( query );
- new ColumnAliasConversionVisitor( lp ).visit();
- new SchemaAliasVisitor( lp ).visit();
- new ScalarVisitor(lp, pc, "test").visit();
-
- CompilationMessageCollector collector = new CompilationMessageCollector() ;
-
- new TypeCheckingRelVisitor( lp, collector).visit();
-
- new UnionOnSchemaSetter( lp ).visit();
- new CastLineageSetter(lp, collector).visit();
+ lp.validate(pc, "test");
return lp;
}
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Arithmetic.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Arithmetic.gld?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Arithmetic.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Arithmetic.gld Fri May 16 18:14:05 2014
@@ -1,4 +1,4 @@
-test-12: Store()
+A: Store()
|
|---A: New For Each(false,false,false)[bag] - scope-64
| |
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/BinCond.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/BinCond.gld?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/BinCond.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/BinCond.gld Fri May 16 18:14:05 2014
@@ -1,4 +1,4 @@
-test-16: Store()
+A: Store()
|
|---A: New For Each(false)[bag] - scope-106
| |
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld Fri May 16 18:14:05 2014
@@ -1,4 +1,4 @@
-test-10: Store()
+A: Store()
|
|---A: Package(Packager)[tuple]{tuple} - scope-19
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Comparison.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Comparison.gld?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Comparison.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Comparison.gld Fri May 16 18:14:05 2014
@@ -1,4 +1,4 @@
-test-14: Store()
+A: Store()
|
|---A: Filter[bag] - scope-67
| |
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld Fri May 16 18:14:05 2014
@@ -1,4 +1,4 @@
-test-1: Store()
+C: Store()
|
|---C: New For Each(false)[bag] - scope-7
| |
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Generate.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Generate.gld?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Generate.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Generate.gld Fri May 16 18:14:05 2014
@@ -1,4 +1,4 @@
-test-18: Store()
+A: Store()
|
|---A: New For Each(false,false,false,false,false,false)[bag] - scope-148
| |
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld Fri May 16 18:14:05 2014
@@ -10,7 +10,7 @@ y: Filter[bag] - scope-170
|
|---test-26: Load()
-test-28: Store()
+x: Store()
|
|---x: Filter[bag] - scope-174
| |
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Split1.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Split1.gld?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Split1.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Split1.gld Fri May 16 18:14:05 2014
@@ -12,7 +12,7 @@ y: Filter[bag] - scope-157
|
|---test-23: Load()
-test-25: Store()
+x: Store()
|
|---x: Filter[bag] - scope-162
| |
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Union.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Union.gld?rev=1595278&r1=1595277&r2=1595278&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Union.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Union.gld Fri May 16 18:14:05 2014
@@ -1,4 +1,4 @@
-test-22: Store()
+A: Store()
|
|---A: Union[bag] - scope-153
|