You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2011/07/21 00:52:48 UTC
svn commit: r1148961 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
test/org/apache/pig/test/TestLimitSchemaStore.java
test/org/apache/pig/test/TestMRCompiler.java
Author: thejas
Date: Wed Jul 20 22:52:45 2011
New Revision: 1148961
URL: http://svn.apache.org/viewvc?rev=1148961&view=rev
Log:
PIG-2146: POStore.getSchema() returns null because of which PigOutputCommitter
is not storing schema while cleanup
Added:
pig/trunk/test/org/apache/pig/test/TestLimitSchemaStore.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1148961&r1=1148960&r2=1148961&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jul 20 22:52:45 2011
@@ -89,6 +89,9 @@ PIG-2011: Speed up TestTypedMap.java (dv
BUG FIXES
+PIG-2146: POStore.getSchema() returns null because of which PigOutputCommitter
+ is not storing schema while cleanup (thejas)
+
PIG-2027: NPE if Pig don't have permission for log file (daijy)
PIG-2171: TestScriptLanguage is broken on trunk (daijy and thejas)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=1148961&r1=1148960&r2=1148961&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Wed Jul 20 22:52:45 2011
@@ -2920,6 +2920,7 @@ public class MRCompiler extends PhyPlanV
POStore st = getStore();
st.setSFile(oldSpec);
st.setIsTmpStore(oldIsTmpStore);
+ st.setSchema(((POStore)mpLeaf).getSchema());
limitAdjustMROp.reducePlan.addAsLeaf(st);
limitAdjustMROp.requestedParallelism = 1;
limitAdjustMROp.setLimitOnly(true);
Added: pig/trunk/test/org/apache/pig/test/TestLimitSchemaStore.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLimitSchemaStore.java?rev=1148961&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLimitSchemaStore.java (added)
+++ pig/trunk/test/org/apache/pig/test/TestLimitSchemaStore.java Wed Jul 20 22:52:45 2011
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.File;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.eclipse.jdt.internal.core.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestLimitSchemaStore{
+
+
+ private PigServer pigServer;
+
+ @Before
+ public void setUp() throws Exception{
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+
+ //PIG-2146
+ @Test //end to end test
+ public void testLimitStoreSchema1() throws Exception{
+ Util.createLocalInputFile("student", new String[]{"joe smith:18:3.5","amy brown:25:2.5","jim fox:20:4.0","leo fu:55:3.0"});
+
+ pigServer.registerQuery("a = load 'student' using " + PigStorage.class.getName() + "(':') as (name, age, gpa);");
+ pigServer.registerQuery("d = distinct a;");
+ pigServer.registerQuery("lim = limit d 1;");
+ String outFile = "limitSchemaOut";
+ Util.deleteDirectory(new File(outFile));
+ pigServer.store("lim", outFile, "PigStorage('\\t', '-schema')");
+ pigServer.dumpSchema("lim");
+
+ pigServer.registerQuery("b = LOAD '" + outFile + "' using PigStorage('\\t', '-schema');");
+ Schema genSchema = pigServer.dumpSchema("b");
+ System.err.println(genSchema);
+ Assert.isNotNull(genSchema);
+
+ }
+
+
+}
Modified: pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java?rev=1148961&r1=1148960&r2=1148961&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMRCompiler.java Wed Jul 20 22:52:45 2011
@@ -55,6 +55,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserComparisonFunc;
import org.apache.pig.impl.plan.NodeIdGenerator;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.test.utils.GenPhyOp;
import org.junit.After;
import org.junit.Before;
@@ -1118,4 +1119,25 @@ public class TestMRCompiler extends juni
assertTrue(mrOper.UDFs.contains(TestIndexableLoadFunc.class.getName()));
}
+
+ //PIG-2146
+ @Test
+ public void testSchemaInStoreForDistinctLimit() throws Exception {
+ //test if the POStore in the 2nd mr plan (that stores the actual output)
+ // has a schema
+ String query = "a = load 'input1' as (a : int,b :float ,c : int);" +
+ "b = distinct a;" +
+ "c = limit b 10;" +
+ "store c into 'output';";
+
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
+ MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
+ MapReduceOper secondMrOper = mrPlan.getLeaves().get(0);
+ POStore store = (POStore)secondMrOper.reducePlan.getLeaves().get(0);
+ assertEquals(
+ "compare load and store schema",
+ store.getSchema(),
+ Utils.getSchemaFromString("a : int,b :float ,c : int")
+ );
+ }
}