You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/04/20 08:03:42 UTC
svn commit: r1095280 - in /pig/branches/branch-0.9: ./
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/newplan/logical/relational/
src/org/apache/pig/newplan/logical/visitor/ test/org/apache/pig/test/
Author: daijy
Date: Wed Apr 20 06:03:41 2011
New Revision: 1095280
URL: http://svn.apache.org/viewvc?rev=1095280&view=rev
Log:
PIG-1962: Wrong alias assinged to store operator
Added:
pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/StoreAliasSetter.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java
pig/branches/branch-0.9/test/org/apache/pig/test/Util.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1095280&r1=1095279&r2=1095280&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Wed Apr 20 06:03:41 2011
@@ -158,6 +158,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1962: Wrong alias assinged to store operator (daijy)
+
PIG-1975: Need to provide backward compatibility for legacy LoadCaster (without bytesToMap(bytes, fieldSchema)) (daijy)
PIG-1987: -dryrun does not work with set (rding)
Modified: pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=1095280&r1=1095279&r2=1095280&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Wed Apr 20 06:03:41 2011
@@ -59,6 +59,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.rules.InputOutputFileValidator;
import org.apache.pig.newplan.logical.rules.LoadStoreFuncDupSignatureValidator;
import org.apache.pig.newplan.logical.visitor.SortInfoSetter;
+import org.apache.pig.newplan.logical.visitor.StoreAliasSetter;
import org.apache.pig.pen.POOptimizeDisabler;
public class HExecutionEngine {
@@ -267,6 +268,9 @@ public class HExecutionEngine {
LoadStoreFuncDupSignatureValidator loadStoreFuncDupSignatureValidator = new LoadStoreFuncDupSignatureValidator(plan);
loadStoreFuncDupSignatureValidator.validate();
+ StoreAliasSetter storeAliasSetter = new StoreAliasSetter( plan );
+ storeAliasSetter.visit();
+
// run optimizer
LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer( plan, 100, optimizerRules );
optimizer.optimize();
Modified: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java?rev=1095280&r1=1095279&r2=1095280&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java Wed Apr 20 06:03:41 2011
@@ -653,8 +653,7 @@ public class LogToPhyTranslationVisitor
// System.err.println("Entering Store");
POStore store = new POStore(new OperatorKey(scope, nodeGen
.getNextNodeId(scope)));
- store.setAlias(((LogicalRelationalOperator)loStore.getPlan().
- getPredecessors(loStore).get(0)).getAlias());
+ store.setAlias(loStore.getAlias());
store.setSFile(loStore.getOutputSpec());
store.setInputSpec(loStore.getInputSpec());
store.setSignature(loStore.getSignature());
Added: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/StoreAliasSetter.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/StoreAliasSetter.java?rev=1095280&view=auto
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/StoreAliasSetter.java (added)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/StoreAliasSetter.java Wed Apr 20 06:03:41 2011
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.newplan.logical.visitor;
+
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.newplan.DependencyOrderWalker;
+import org.apache.pig.newplan.OperatorPlan;
+import org.apache.pig.newplan.logical.relational.LOStore;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
+
+public class StoreAliasSetter extends LogicalRelationalNodesVisitor {
+
+ public StoreAliasSetter(OperatorPlan plan) throws FrontendException {
+ super(plan, new DependencyOrderWalker(plan));
+ }
+
+ @Override
+ public void visit(LOStore store) throws FrontendException {
+ store.setAlias(((LogicalRelationalOperator)plan.getPredecessors(store).get(0)).getAlias());
+ }
+}
Added: pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java?rev=1095280&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java (added)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java Wed Apr 20 06:03:41 2011
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.util.Properties;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.newplan.logical.relational.LOStore;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestPlanGeneration extends junit.framework.TestCase {
+
+ PigContext pc;
+
+ @Override
+ @BeforeClass
+ public void setUp() throws ExecException {
+ pc = new PigContext(ExecType.LOCAL, new Properties());
+ pc.connect();
+ }
+
+ @Test
+ public void testStoreAlias() throws Exception {
+ String query = "A = load 'data' as (a0, a1);" +
+ "B = filter A by a0 > 1;" +
+ "store B into 'output';";
+
+ LogicalPlan lp = Util.parse(query, pc);
+ Util.optimizeNewLP(lp);
+ LOStore loStore = (LOStore)lp.getSinks().get(0);
+ assert(loStore.getAlias().equals("B"));
+
+ PhysicalPlan pp = Util.buildPhysicalPlanFromNewLP(lp, pc);
+ POStore poStore = (POStore)pp.getLeaves().get(0);
+ assert(poStore.getAlias().equals("B"));
+
+ MROperPlan mrp = Util.buildMRPlanWithOptimizer(pp, pc);
+ MapReduceOper mrOper = mrp.getLeaves().get(0);
+ poStore = (POStore)mrOper.mapPlan.getLeaves().get(0);
+ assert(poStore.getAlias().equals("B"));
+ }
+}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/Util.java?rev=1095280&r1=1095279&r2=1095280&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/Util.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/Util.java Wed Apr 20 06:03:41 2011
@@ -81,10 +81,22 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.parser.QueryParser;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.plan.CompilationMessageCollector;
import org.apache.pig.impl.util.LogUtils;
import org.apache.pig.newplan.logical.LogicalPlanMigrationVistor;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanPrinter;
import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
+import org.apache.pig.newplan.logical.optimizer.UidResetter;
+import org.apache.pig.newplan.logical.rules.LoadStoreFuncDupSignatureValidator;
+import org.apache.pig.newplan.logical.visitor.CastLineageSetter;
+import org.apache.pig.newplan.logical.visitor.ColumnAliasConversionVisitor;
+import org.apache.pig.newplan.logical.visitor.ScalarVisitor;
+import org.apache.pig.newplan.logical.visitor.SchemaAliasVisitor;
+import org.apache.pig.newplan.logical.visitor.SortInfoSetter;
+import org.apache.pig.newplan.logical.visitor.StoreAliasSetter;
+import org.apache.pig.newplan.logical.visitor.TypeCheckingRelVisitor;
+import org.apache.pig.newplan.logical.visitor.UnionOnSchemaSetter;
+import org.apache.pig.parser.QueryParserDriver;
import org.apache.pig.tools.grunt.GruntParser;
import com.google.common.base.Function;
@@ -670,10 +682,26 @@ public class Util {
public static org.apache.pig.newplan.logical.relational.LogicalPlan optimizeNewLP(
org.apache.pig.newplan.logical.relational.LogicalPlan lp)
throws FrontendException{
+ UidResetter uidResetter = new UidResetter( lp );
+ uidResetter.visit();
+
+ SchemaResetter schemaResetter = new SchemaResetter( lp );
+ schemaResetter.visit();
+
+ LoadStoreFuncDupSignatureValidator loadStoreFuncDupSignatureValidator = new LoadStoreFuncDupSignatureValidator(lp);
+ loadStoreFuncDupSignatureValidator.validate();
+
+ StoreAliasSetter storeAliasSetter = new StoreAliasSetter( lp );
+ storeAliasSetter.visit();
+
// run optimizer
org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer optimizer =
new org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer(lp, 100, null);
- optimizer.optimize();
+ optimizer.optimize();
+
+ SortInfoSetter sortInfoSetter = new SortInfoSetter( lp );
+ sortInfoSetter.visit();
+
return lp;
}
@@ -901,4 +929,22 @@ public class Util {
fail("caught exception while checking log message :" + e);
}
}
+
+ public static org.apache.pig.newplan.logical.relational.LogicalPlan parse(String query, PigContext pc) throws FrontendException {
+ Map<String, String> fileNameMap = new HashMap<String, String>();
+ QueryParserDriver parserDriver = new QueryParserDriver( pc, "test", fileNameMap );
+ org.apache.pig.newplan.logical.relational.LogicalPlan lp = parserDriver.parse( query );
+
+ new ColumnAliasConversionVisitor( lp ).visit();
+ new SchemaAliasVisitor( lp ).visit();
+ new ScalarVisitor( lp, pc ).visit();
+
+ CompilationMessageCollector collector = new CompilationMessageCollector() ;
+
+ new TypeCheckingRelVisitor( lp, collector).visit();
+
+ new UnionOnSchemaSetter( lp ).visit();
+ new CastLineageSetter(lp, collector).visit();
+ return lp;
+ }
}