You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2011/03/28 14:24:40 UTC
svn commit: r1086206 [3/3] - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/
src/org/apache/pig/backend/hadoop/executionengine/phys...
Added: pig/trunk/test/org/apache/pig/test/TestProjectRange.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestProjectRange.java?rev=1086206&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestProjectRange.java (added)
+++ pig/trunk/test/org/apache/pig/test/TestProjectRange.java Mon Mar 28 12:24:38 2011
@@ -0,0 +1,1265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+import static org.apache.pig.ExecType.LOCAL;
+import static org.apache.pig.ExecType.MAPREDUCE;
+import static org.junit.Assert.assertEquals;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.parser.ParseException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+import org.apache.pig.impl.plan.CompilationMessageCollector;
+import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
+import org.apache.pig.newplan.logical.relational.LOCogroup;
+import org.apache.pig.newplan.logical.relational.LOJoin;
+import org.apache.pig.newplan.logical.relational.LOSort;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.visitor.CastLineageSetter;
+import org.apache.pig.newplan.logical.visitor.ColumnAliasConversionVisitor;
+import org.apache.pig.newplan.logical.visitor.SchemaAliasVisitor;
+import org.apache.pig.newplan.logical.visitor.TypeCheckingRelVisitor;
+import org.apache.pig.newplan.logical.visitor.UnionOnSchemaSetter;
+import org.apache.pig.parser.ParserTestingUtils;
+import org.apache.pig.test.utils.NewLogicalPlanUtil;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestProjectRange {
+
+ protected final Log log = LogFactory.getLog(getClass());
+
+ protected static ExecType execType = LOCAL;
+
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static final String INP_FILE_5FIELDS = "TestProjectRange_5fields";
+
+
+ @BeforeClass
+ public static void oneTimeSetUp() throws Exception {
+
+ String execTypeString = System.getProperty("test.exectype");
+
+ if(execTypeString!=null && execTypeString.length()>0){
+ execType = PigServer.parseExecType(execTypeString);
+ }
+
+ PrintWriter w = new PrintWriter(new FileWriter(INP_FILE_5FIELDS));
+ w.println("10\t20\t30\t40\t50");
+ w.println("11\t21\t31\t41\t51");
+ w.close();
+
+ if(execType == MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ Util.copyFromLocalToCluster(cluster, INP_FILE_5FIELDS, INP_FILE_5FIELDS);
+ }
+ }
+
+ @Before
+ public void setup() throws ExecException{
+ if(execType == MAPREDUCE) {
+ pigServer = new PigServer(MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(LOCAL);
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ @AfterClass
+ public static void oneTimeTearDown() throws Exception {
+
+ if(cluster != null)
+ cluster.shutDown();
+ }
+
+
+ /**
+ * Test project-range in foreach with limits on both sides
+ * @throws IOException
+ * @throws ParseException
+ */
+
+ @Test
+ public void testFullRangeForeachWSchema() throws IOException, ParseException {
+
+ String query;
+
+ //specifying the new aliases
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate a .. c as (aa, bb, cc);"
+ ;
+ compileAndCompareSchema("aa : int, bb : float, cc : int", query, "f");
+
+ //specifying the new aliases - refer to column by pos
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate $0 .. $2 as (aa, bb, cc);"
+ ;
+ compileAndCompareSchema("aa : int, bb : float, cc : int", query, "f");
+
+ //column with pos , name
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate $0 .. c as (aa, bb, cc);"
+ ;
+ compileAndCompareSchema("aa : int, bb : float, cc : int", query, "f");
+
+
+ //specifying the new aliases
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate b .. d as (bb, cc, dd);"
+ ;
+ compileAndCompareSchema("bb : float, cc : int, dd : int", query, "f");
+
+ //begin, end of range is same
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate b .. b as (bb), $2 .. $2;"
+ ;
+ compileAndCompareSchema("bb : float, c : int", query, "f");
+
+ // without aliases - two projections
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int, d : int, e : int);"
+ + "f = foreach l1 generate a .. c, d .. e ;"
+ ;
+ compileAndCompareSchema("a : int, b : int, c : int, d : int, e : int", query, "f");
+
+ // without aliases
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int, d : int, e : int);"
+ + "f = foreach l1 generate a .. c ;"
+ ;
+ compileAndCompareSchema("a : int, b : int, c : int", query, "f");
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(10,20,30)",
+ "(11,21,31)",
+ });
+ Util.checkQueryOutputsAfterSort(it, expectedRes);
+
+ }
+
+
+ private void compileAndCompareSchema(String expectedSchStr, String query, String alias)
+ throws IOException, ParseException {
+
+ Schema expectedSch = null;
+
+ if(expectedSchStr != null)
+ expectedSch = Util.getSchemaFromString(expectedSchStr);
+
+ compileAndCompareSchema(expectedSch, query, alias);
+
+ }
+
+ private void compileAndCompareSchema(Schema expectedSch, String query,
+ String alias) throws IOException {
+ Util.registerMultiLineQuery(pigServer, query);
+
+ Schema sch = pigServer.dumpSchema(alias);
+ assertEquals("Checking expected schema", expectedSch, sch);
+ }
+
+ /**
+ * Test project-range in foreach with starting limit
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testEndRangeForeachWSchema() throws IOException, ParseException {
+
+ //specifying the new aliases
+ String query;
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate .. c as (aa, bb, cc);"
+ ;
+ compileAndCompareSchema("aa : int, bb : float, cc : int", query, "f");
+
+ //col position
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate .. $2 as (aa, bb, cc);"
+ ;
+ compileAndCompareSchema("aa : int, bb : float, cc : int", query, "f");
+
+ //end is the beginning!
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate .. $0 as (aa, bb, cc);"
+ ;
+ compileAndCompareSchema("aa : int", query, "f");
+
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate .. c as (aa, bb, cc);"
+ ;
+ compileAndCompareSchema("aa : int, bb : float, cc : int", query, "f");
+
+ // without aliases
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int, d : long, e : int);"
+ + "f = foreach l1 generate .. $3 ;"
+ ;
+ compileAndCompareSchema("a : int, b : int, c : int, d : long", query, "f");
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(10,20,30,40L)",
+ "(11,21,31,41L)",
+ });
+ Util.checkQueryOutputsAfterSort(it, expectedRes);
+
+ }
+
+
+ /**
+ * Test project-range in foreach with start limit
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testStartRangeForeachWSchema() throws IOException, ParseException {
+
+ //specifying the new aliases
+ String query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' " +
+ "as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate c .. as (aa, bb, cc);"
+ ;
+ compileAndCompareSchema("aa : int, bb : int, cc : int", query, "f");
+
+ // without aliases
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' " +
+ "as (a : int, b : int, c : int, d : long, e : int);"
+ + "f = foreach l1 generate $1 .. ;"
+ ;
+ compileAndCompareSchema("b : int, c : int, d : long, e : int", query, "f");
+
+ //start with last column - beginning is the end!
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' " +
+ "as (a : int, b : int, c : int, d : long, e : int);"
+ + "f = foreach l1 generate e .. ;"
+ ;
+ compileAndCompareSchema("e : int", query, "f");
+
+ //specifying the new aliases for one
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' " +
+ "as (a : int, b : long, c : int, d : int, e : int);"
+ + "f = foreach l1 generate c .. as (aa, bb, cc), b .. ;"
+ ;
+ compileAndCompareSchema(
+ "aa : int, bb : int, cc : int, b : long, c : int, d : int, e : int",
+ query,
+ "f"
+ );
+
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(30,40,50,20L,30,40,50)",
+ "(31,41,51,21L,31,41,51)",
+
+ });
+ Util.checkQueryOutputsAfterSort(it, expectedRes);
+
+ }
+
+
+ /**
+ * Test multiple different types of range-project with foreach
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testMixRangeForeachWSchema() throws IOException, ParseException {
+
+ //specifying the new aliases
+ String query;
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate .. b, c .. d, d .. as (aa, bb);"
+ ;
+ compileAndCompareSchema("a : int, b : float, c : int, d : int, aa : int, bb : int", query, "f");
+
+
+ // without aliases
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int, d : long, e : int);"
+ + "f = foreach l1 generate ..$0 as (first), e.. as (last), d .., .. b ;"
+ ;
+ compileAndCompareSchema("first : int, last : int, d : long, e : int, a : int, b : int", query, "f");
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(10,50,40L,50,10,20)",
+ "(11,51,41L,51,11,21)",
+ });
+ Util.checkQueryOutputsAfterSort(it, expectedRes);
+
+ }
+
+ /**
+ * -ve test cases
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testNegativeForeachWSchema() throws IOException, ParseException {
+ String query;
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate $3 .. $1;"
+ ;
+ Util.checkExceptionMessage(query, "f",
+ "start column appears after end column in range projection");
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : float, c : int, d : int, e : int);"
+ + "f = foreach l1 generate c .. b;"
+ ;
+ Util.checkExceptionMessage(query, "f",
+ "start column appears after end column in range projection");
+ }
+
+
+ /**
+ * -ve test cases
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testNegativeForeachNOSchema() throws IOException, ParseException {
+ String query;
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + "f = foreach l1 generate $3 .. $1;"
+ ;
+ Util.checkExceptionMessage(query, "f",
+ "start column appears after end column in range projection");
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' ;"
+ + "f = foreach l1 generate a .. b;"
+ ;
+ Util.checkExceptionMessage(query, "f",
+ "Invalid field projection. Projected field [a] does not exist in schema");
+ }
+
+ /**
+ * Test foreach without schema
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testStartRangeForeachNOSchema() throws IOException, ParseException {
+
+ String query;
+
+ // without aliases
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + "f = foreach l1 generate ..$3 ;"
+ ;
+ // the schema should be null, but that is not the case
+ // see - PIG-1910
+ //compileAndCompareSchema(null, query, "f");
+
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ pigServer.explain("f", System.err);
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStringAsByteArray(
+ new String[] {
+ "(10,20,30,40)",
+ "(11,21,31,41)",
+ });
+ Util.checkQueryOutputsAfterSort(it, expectedRes);
+
+ }
+
+
+ /**
+ * Test foreach without schema
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testMixRangeForeachNOSchema() throws IOException, ParseException {
+
+ String query;
+
+ // without aliases
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + "f = foreach l1 generate ..$0 as (first), $4.. as (last), $3 .., .. $1 ;"
+ ;
+ // the schema should be null, but that is not the case
+ // see - PIG-1910
+ //compileAndCompareSchema(null, query, "f");
+
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ pigServer.explain("f", System.err);
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStringAsByteArray(
+ new String[] {
+ "(10,50,40,50,10,20)",
+ "(11,51,41,51,11,21)",
+ });
+ Util.checkQueryOutputsAfterSort(it, expectedRes);
+
+ }
+
+ /**
+ * Test foreach without schema - with some operations after the foreach
+ * @throws IOException
+ * @throws ParseException
+ */
+ //@Test
+ public void testRangeForeachWFilterNOSchema() throws IOException, ParseException {
+ //TODO: fix depends on PIG-1910
+ //fails with - In alias fil, incompatible types in GreaterThan
+ // Operator left hand side:Unknown right hand side:Unknown
+ String query;
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + "f = foreach l1 generate ..$0 as (first), $4.. as (last), $3 .., .. $1 ;"
+ + " fil = filter f by $0 > 10;"
+ ;
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ pigServer.explain("fil", System.err);
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStringAsByteArray(
+ new String[] {
+ "(11,51,41,51,11,21)",
+ });
+ Util.checkQueryOutputsAfterSort(it, expectedRes);
+
+ }
+
+ @Test
+ public void testRangeOrderByWSchema() throws IOException, ParseException{
+ String query;
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " o = order l1 by .. $2 DESC ;"
+ ;
+ compileAndCompareSchema("a : int, b : long, c : int, d : int, e : int", query, "o");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {false,false,false};
+ checkNumExpressionPlansForSort(lp, 3, isAsc);
+ }
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " o = order l1 by $3 .. ;"
+ ;
+ compileAndCompareSchema("a : int, b : long, c : int, d : int, e : int", query, "o");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {true, true};
+ checkNumExpressionPlansForSort(lp, 2, isAsc);
+ }
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " o = order l1 by d .. DESC ;"
+ ;
+ compileAndCompareSchema("a : int, b : long, c : int, d : int, e : int", query, "o");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {false, false};
+ checkNumExpressionPlansForSort(lp, 2, isAsc);
+ }
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " f = foreach l1 generate *;"
+ + " o = order f by $0 .. c ASC ;"
+ + " lim = limit o 10; ;"
+ ;
+ compileAndCompareSchema("a : int, b : long, c : int, d : int, e : int", query, "lim");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {true, true, true};
+ checkNumExpressionPlansForSort(lp, 3, isAsc);
+ }
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " o = order l1 by $0 .. $4 ;"
+ ;
+ compileAndCompareSchema("a : int, b : long, c : int, d : int, e : int", query, "o");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {true,true,true,true,true};
+ checkNumExpressionPlansForSort(lp, 5, isAsc);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ Iterator<Tuple> it = pigServer.openIterator("o");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(10,20,30,40,50)",
+ "(11,21,31,41,51)",
+ });
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+
+ /**
+ * Test nested order-by with schema
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testRangeOrderByNestedWSchema() throws IOException, ParseException{
+ String query;
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " g = group l1 by a;"
+ + " f = foreach g { o = order l1 by .. $2 DESC; generate group, o;}"
+ ;
+ String expectedSchStr = "g : int,o: {t : (a: int,b: long,c: int,d: int,e: int)}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "f");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {false,false,false};
+ checkNumExpressionPlansForSort(lp, 3, isAsc);
+ }
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " g = group l1 by a;"
+ + " f = foreach g { o = order l1 by d .. ; generate group, o;}"
+ ;
+ String expectedSchStr = "g : int,o: {t : (a: int,b: long,c: int,d: int,e: int)}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "f");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {true,true};
+ checkNumExpressionPlansForSort(lp, 2, isAsc);
+ }
+ {
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " g = group l1 by a;"
+ + " f = foreach g { o = order l1 by $2 .. $3 ASC, $1..c DESC; generate group, o;}"
+ ;
+ String expectedSchStr = "g : int,o: {t : (a: int,b: long,c: int,d: int,e: int)}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "f");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {true,true,false,false};
+ checkNumExpressionPlansForSort(lp, 4, isAsc);
+
+ }
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " g = group l1 by a;"
+ + " f = foreach g { o = order l1 by $2 .. $3 DESC, $1 ASC; generate group, o;}"
+ ;
+
+ String expectedSchStr = "g : int,o: {t : (a: int,b: long,c: int,d: int,e: int)}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "f");
+
+ //check number of sort expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {false,false,true};
+ checkNumExpressionPlansForSort(lp, 3, isAsc);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(10,{(10,20,30,40,50)})",
+ "(11,{(11,21,31,41,51)})",
+ });
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+ /**
+ * Test nested order-by without schema
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testRangeOrderByNestedNOSchema() throws IOException, ParseException{
+ String query;
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + " g = group l1 by $0;"
+ + " f = foreach g { o = order l1 by .. $2 DESC; generate group, o;}"
+ ;
+ String expectedSchStr = "g : bytearray, o: {t : ()}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "f");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {false,false,false};
+ checkNumExpressionPlansForSort(lp, 3, isAsc);
+ }
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + " g = group l1 by $0;"
+ + " f = foreach g { o = order l1 by $3 .. ; generate group, o;}"
+ ;
+ String expectedSchStr = "g : bytearray, o: {t : ()}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "f");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ //project to end can't be expanded
+ boolean[] isAsc = {true};
+ checkNumExpressionPlansForSort(lp, 1, isAsc);
+ }
+ {
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + " g = group l1 by $1;"
+ + " f = foreach g { o = order l1 by $2 .. $3 ASC, $1 .. $2 DESC; generate group, o;}"
+ ;
+ String expectedSchStr = "g : bytearray, o: {t : ()}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "f");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {true,true,false,false};
+ checkNumExpressionPlansForSort(lp, 4, isAsc);
+
+ }
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + " g = group l1 by 1;"
+ + " f = foreach g { o = order l1 by $2 .. $3 desc; generate group, o;}"
+ ;
+ String expectedSchStr = "g : int, o: {t : ()}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "f");
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {false,false};
+ checkNumExpressionPlansForSort(lp, 2, isAsc);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ Iterator<Tuple> it = pigServer.openIterator("f");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(1,{(11,21,31,41,51),(10,20,30,40,50)})",
+ });
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+ private LOSort checkNumExpressionPlansForSort(LogicalPlan lp, int numPlans, boolean[] isAsc) {
+ Class<?> sortClass = org.apache.pig.newplan.logical.relational.LOSort.class;
+ LOSort sort = (LOSort) NewLogicalPlanUtil.getRelOpFromPlan(lp, sortClass);
+ assertEquals("number of sort col plans", numPlans, sort.getSortColPlans().size());
+
+ List<Boolean> ascCols = sort.getAscendingCols();
+ for(int i = 0; i < ascCols.size(); i++){
+ assertEquals("ascending order", isAsc[i], ascCols.get(i));
+ }
+
+ return sort;
+ }
+
+ private LogicalPlan createAndProcessLPlan(String query) throws FrontendException {
+ //TODO: create a common util function for logical plan tests
+ LogicalPlan lp = generateLogicalPlan(query);
+ new ColumnAliasConversionVisitor( lp ).visit();
+ new SchemaAliasVisitor( lp ).visit();
+
+ CompilationMessageCollector collector = new CompilationMessageCollector() ;
+ new TypeCheckingRelVisitor( lp, collector).visit();
+ new UnionOnSchemaSetter( lp ).visit();
+ new CastLineageSetter(lp, collector).visit();
+
+ return lp;
+
+ }
+
+ private LogicalPlan generateLogicalPlan(String query) {
+ try {
+ return ParserTestingUtils.generateLogicalPlan( query );
+ } catch(Exception ex) {
+ ex.printStackTrace();
+ Assert.fail( "Failed to generate logical plan for query [" + query + "] due to exception: " + ex );
+ }
+ return null;
+ }
+
+ @Test
+ public void testRangeOrderByMixWSchema() throws IOException, ParseException{
+ String query;
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);"
+ + " o = order l1 by b .. c, a DESC, d .. DESC;"
+ ;
+ compileAndCompareSchema("a : int, b : long, c : int, d : int, e : int", query, "o");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {true,true,false,false,false};
+ checkNumExpressionPlansForSort(lp, 5, isAsc);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ Iterator<Tuple> it = pigServer.openIterator("o");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(10,20,30,40,50)",
+ "(11,21,31,41,51)",
+ });
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+
+ @Test
+ public void testRangeOrderByMixNOSchema() throws IOException, ParseException{
+ String query;
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + " o = order l1 by $1 .. $2 DESC, $0 , $4 .. DESC;"
+ ;
+ compileAndCompareSchema((Schema)null, query, "o");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {false, false,true,false};
+ checkNumExpressionPlansForSort(lp, 4, isAsc);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ pigServer.explain("o", System.err);
+ Iterator<Tuple> it = pigServer.openIterator("o");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(11,21,31,41,51)",
+ "(10,20,30,40,50)",
+ });
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+ @Test
+ public void testRangeOrderByStartNOSchema() throws IOException, ParseException{
+ String query;
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + " o = order l1 by $3 .. DESC;"
+ ;
+ compileAndCompareSchema((Schema)null, query, "o");
+
+ //check number of sort expression plans
+
+ LogicalPlan lp = createAndProcessLPlan(query);
+ boolean[] isAsc = {false};
+ checkNumExpressionPlansForSort(lp, 1, isAsc);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ pigServer.explain("o", System.err);
+ Iterator<Tuple> it = pigServer.openIterator("o");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(11,21,31,41,51)",
+ "(10,20,30,40,50)",
+ });
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+ @Test
+ public void testRangeOrderByStartNegNOSchema() throws IOException, ParseException{
+ String query;
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';"
+ + " o = order l1 by $3 .. DESC, $1;"
+ ;
+ Util.checkExceptionMessage(query, "o","Project-range to end (eg. x..)" +
+ " is supported in order-by only as last sort column");
+ }
+
+ @Test
+ public void testRangeGroupWSchema() throws IOException, ParseException{
+ String query;
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " l2 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " g = group l1 by d .., l2 by d ..;"
+ ;
+ String expectedSchStr = "grp: (d: int,e : int)," +
+ "l1: {t : (a: int,b: long,c: int,d: int,e: int)}," +
+ "l2: {t : (a: int,b: long,c: int,d: int,e: int)}";
+
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "g");
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForGroup(lp, 2);
+ }
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " l2 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " g = group l1 by c .. $3, l2 by $3..$4;"
+ ;
+ String expectedSchStr = "grp: (c: int,d : int)," +
+ "l1: {t : (a: int,b: long,c: int,d: int,e: int)}," +
+ "l2: {t : (a: int,b: long,c: int,d: int,e: int)}";
+
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "g");
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForGroup(lp, 2);
+ }
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " l2 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " g = group l1 by .. c, l2 by .. c;"
+ ;
+ String expectedSchStr = "grp: (a: int,b: long,c: int)," +
+ "l1: {t : (a: int,b: long,c: int,d: int,e: int)}," +
+ "l2: {t : (a: int,b: long,c: int,d: int,e: int)}";
+
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "g");
+
+
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForGroup(lp, 3);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "((10,20,30),{(10,20,30,40,50)},{(10,20,30,40,50)})",
+ "((11,21,31),{(11,21,31,41,51)},{(11,21,31,41,51)})",
+ });
+ Iterator<Tuple> it = pigServer.openIterator("g");
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+ /**
+ * some transformations to schema, because the parser does not accept
+ * group as a column name in schema, and to add tuple within bag schema
+ * @param expectedSchStr
+ * @return
+ * @throws ParseException
+ * @throws FrontendException
+ */
+ private Schema getCleanedGroupSchema(String expectedSchStr) throws ParseException, FrontendException {
+ Schema expectedSch = Util.getSchemaFromString(expectedSchStr);
+ expectedSch.getField(0).alias = "group";
+ if(expectedSch.size() > 1)
+ expectedSch.getField(1).schema.getField(0).alias = null;
+ if(expectedSch.size() > 2)
+ expectedSch.getField(2).schema.getField(0).alias = null;
+ expectedSch = org.apache.pig.newplan.logical.Util.fixSchemaAddTupleInBag(expectedSch);
+ return expectedSch;
+ }
+
+ private LOCogroup checkNumExpressionPlansForGroup(LogicalPlan lp, int numPlans) {
+ Class<?> groupClass = org.apache.pig.newplan.logical.relational.LOCogroup.class;
+ LOCogroup grp = (LOCogroup) NewLogicalPlanUtil.getRelOpFromPlan(lp, groupClass);
+
+ for( int inp : grp.getExpressionPlans().keySet()){
+ List<LogicalExpressionPlan> plans = grp.getExpressionPlans().get(inp);
+ assertEquals("number of group-by plans", numPlans, plans.size());
+ }
+
+ return grp;
+ }
+
+ @Test
+ public void testRangeCoGroupMixWSchema() throws IOException, ParseException{
+ String query;
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " l2 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " g = group l1 by (a + b, c .. d, e.. ), l2 by ($0 + $1, c..d, $4..);"
+ ;
+ String expectedSchStr = "grp: (x : long, c :int , d :int, e : int)," +
+ "l1: {t : (a: int,b: long,c: int,d: int,e: int)}," +
+ "l2: {t : (a: int,b: long,c: int,d: int,e: int)}";
+
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ expectedSch.getField(0).schema.getField(0).alias = null;
+ compileAndCompareSchema(expectedSch, query, "g");
+
+
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForGroup(lp, 4);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "((30,30,40,50),{(10,20,30,40,50)},{(10,20,30,40,50)})",
+ "((32,31,41,51),{(11,21,31,41,51)},{(11,21,31,41,51)})",
+ });
+ Iterator<Tuple> it = pigServer.openIterator("g");
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+ @Test
+ public void testRangeGroupMixWSchema() throws IOException, ParseException{
+ String query;
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " g = group l1 by b .. c;"
+ ;
+ String expectedSchStr = "grp: (b : long, c :int)," +
+ "l1: {t : (a: int,b: long,c: int,d: int,e: int)}";
+
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "g");
+
+
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForGroup(lp, 2);
+ }
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " g = group l1 by $2 .. ;" +
+ " lim = limit g 2;"
+ ;
+ String expectedSchStr = "grp: (c :int , d :int, e : int)," +
+ "l1: {t : (a: int,b: long,c: int,d: int,e: int)}";
+
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "lim");
+
+
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForGroup(lp, 3);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "((30,40,50),{(10,20,30,40,50)})",
+ "((31,41,51),{(11,21,31,41,51)})",
+ });
+ Iterator<Tuple> it = pigServer.openIterator("lim");
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+
+ @Test
+ public void testRangeGroupMixNOSchema() throws IOException, ParseException{
+ String query;
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';" +
+ " g = group l1 by .. $2;"
+ ;
+ String expectedSchStr = "g : (duma, dumb, dumc), l1: {t : ()}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ setAliasesToNull(expectedSch.getField(0).schema);
+ compileAndCompareSchema(expectedSch, query, "g");
+
+
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForGroup(lp, 3);
+ }
+
+ {
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';" +
+ " g = group l1 by $3 .. $3;"
+ ;
+ String expectedSchStr = "g : bytearray, l1: {t : ()}";
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "g");
+
+
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForGroup(lp, 1);
+ }
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';" +
+ " g = group l1 by $2 .. ;"
+ ;
+ String expectedSchStr = "grp: (), l1: {t : ()}";
+
+ Schema expectedSch = getCleanedGroupSchema(expectedSchStr);
+ compileAndCompareSchema(expectedSch, query, "g");
+
+
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForGroup(lp, 1);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "((30,40,50),{(10,20,30,40,50)})",
+ "((31,41,51),{(11,21,31,41,51)})",
+ });
+ Iterator<Tuple> it = pigServer.openIterator("g");
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+ private void setAliasesToNull(Schema schema) {
+ for(FieldSchema fs : schema.getFields()){
+ fs.alias = null;
+ }
+ }
+
+ @Test
+ public void testRangeJoinMixWSchema() throws IOException, ParseException{
+ String query;
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " l2 = load '" + INP_FILE_5FIELDS +
+ "' as (a : int, b : long, c : int, d : int, e : int);" +
+ " j = join l1 by (a + b, c .. d, e.. ), l2 by ($0 + $1, c..d, $4..);"
+ ;
+ String expectedSchStr = "l1::a: int, l1::b: long, l1::c: int, l1::d: int, l1::e: int," +
+ "l2::a: int, l2::b: long, l2::c: int, l2::d: int, l2::e: int";
+
+ compileAndCompareSchema(expectedSchStr, query, "j");
+
+
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForJoin(lp, 4);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(10,20,30,40,50,10,20,30,40,50)",
+ "(11,21,31,41,51,11,21,31,41,51)",
+ });
+ Iterator<Tuple> it = pigServer.openIterator("j");
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+ @Test
+ public void testRangeJoinMixNOSchema() throws IOException, ParseException{
+ String query;
+
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';" +
+ " l2 = load '" + INP_FILE_5FIELDS + "';" +
+ " j = join l1 by $0 .. $3, l2 by $0 .. $3;"
+ ;
+
+ compileAndCompareSchema((Schema)null, query, "j");
+
+ //check number of group expression plans
+ LogicalPlan lp = createAndProcessLPlan(query);
+ checkNumExpressionPlansForJoin(lp, 4);
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(10,20,30,40,50,10,20,30,40,50)",
+ "(11,21,31,41,51,11,21,31,41,51)",
+ });
+ Iterator<Tuple> it = pigServer.openIterator("j");
+ Util.checkQueryOutputs(it, expectedRes);
+ }
+
+ @Test
+ public void testRangeCoGroupNegNoSchema() throws IOException, ParseException{
+ String query;
+ //cogroup
+ query =
+ " l1 = load '" + INP_FILE_5FIELDS + "';" +
+ " l2 = load '" + INP_FILE_5FIELDS + "';" +
+ " g = cogroup l1 by ($0 .. ), l2 by ($0 .. );";
+ Util.checkExceptionMessage(query, "g", "Cogroup/Group by '*' or 'x..' " +
+ "(range of columns to the end) " +
+ "is only allowed if the input has a schema");
+ }
+
+ private LOJoin checkNumExpressionPlansForJoin(LogicalPlan lp, int numPlans) {
+ Class<?> joinClass = org.apache.pig.newplan.logical.relational.LOJoin.class;
+ LOJoin join = (LOJoin) NewLogicalPlanUtil.getRelOpFromPlan(lp, joinClass);
+
+ for( int inp : join.getExpressionPlans().keySet()){
+ List<LogicalExpressionPlan> plans = join.getExpressionPlans().get(inp);
+ assertEquals("number of join exp plans", numPlans, plans.size());
+ }
+
+ return join;
+ }
+
+}
Modified: pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1086206&r1=1086205&r2=1086206&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Mon Mar 28 12:24:38 2011
@@ -114,6 +114,7 @@ import org.apache.pig.parser.ParserTesti
import org.apache.pig.test.utils.LogicalPlanTester;
import org.junit.Before;
import org.junit.Test;
+import static org.apache.pig.test.Util.*;
public class TestTypeCheckingValidatorNewLP {
@@ -2711,16 +2712,13 @@ public class TestTypeCheckingValidatorNe
String query = "a = load 'a' using PigStorage('a') ;"
+ "b = load 'b' using PigStorage('b') ;"
+ "c = cogroup a by *, b by *;";
- boolean exceptionThrown = false;
- try {
- ParserTestingUtils.generateLogicalPlan( query );
- } catch(Exception e) {
- assertTrue(e.toString().contains("Cogroup/Group by * is only allowed if " +
- "the input has a schema"));
- exceptionThrown = true;
- }
- assertTrue(exceptionThrown);
-
+
+ String exMsg= "Cogroup/Group by '*' or 'x..' " +
+ "(range of columns to the end) " +
+ "is only allowed if the input has a schema";
+
+ checkExceptionMessage(query, "c", exMsg);
+
}
@Test
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1086206&r1=1086205&r2=1086206&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Mon Mar 28 12:24:38 2011
@@ -58,6 +58,7 @@ import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.SimpleLayout;
import org.apache.pig.ExecType;
+import org.apache.pig.PigException;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
@@ -80,6 +81,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.parser.QueryParser;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.LogUtils;
import org.apache.pig.newplan.logical.LogicalPlanMigrationVistor;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanPrinter;
import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
@@ -407,7 +409,7 @@ public class Util {
* expected results.
*
* @param actualResults Result of the executed Pig query
- * @param expectedResults Expected results to validate against
+ * @param expectedResults Expected results Array to validate against
*/
static public void checkQueryOutputs(Iterator<Tuple> actualResults,
Tuple[] expectedResults) {
@@ -416,6 +418,19 @@ public class Util {
Assert.assertEquals(expected.toString(), actual.toString());
}
}
+
+ /**
+ * Helper function to check if the result of a Pig Query is in line with
+ * expected results.
+ *
+ * @param actualResults Result of the executed Pig query
+ * @param expectedResults Expected results List to validate against
+ */
+ static public void checkQueryOutputs(Iterator<Tuple> actualResults,
+ List<Tuple> expectedResults) {
+
+ checkQueryOutputs(actualResults,expectedResults.toArray(new Tuple[expectedResults.size()]));
+ }
/**
* Helper function to check if the result of a Pig Query is in line with
@@ -448,6 +463,37 @@ public class Util {
fail("String '"+ subStr + "' is not a substring of '" + str + "'");
}
}
+
+ /**
+ * Check if query plan for alias argument produces exception with expected
+ * error message in expectedErr argument.
+ * @param query
+ * @param alias
+ * @param expectedErr
+ * @throws IOException
+ */
+ static public void checkExceptionMessage(String query, String alias, String expectedErr)
+ throws IOException {
+ PigServer pig = new PigServer(ExecType.LOCAL);
+
+ boolean foundEx = false;
+ try{
+ Util.registerMultiLineQuery(pig, query);
+ pig.explain(alias, System.out);
+ }catch(FrontendException e){
+ PigException pigEx = LogUtils.getPigException(e);
+ foundEx = true;
+ if(!pigEx.getMessage().contains(expectedErr)){
+ String msg = "Expected exception message matching '"
+ + expectedErr + "' but got '" + pigEx.getMessage() + "'" ;
+ fail(msg);
+ }
+ }
+
+ if(!foundEx)
+ fail("No exception thrown. Exception is expected.");
+
+ }
/**
* Utility method to copy a file form local filesystem to the dfs on
@@ -563,6 +609,19 @@ public class Util {
return result;
}
+ public static List<Tuple> getTuplesFromConstantTupleStringAsByteArray(String[] tupleConstants)
+ throws ParseException, ExecException{
+ List<Tuple> tuples = getTuplesFromConstantTupleStrings(tupleConstants);
+ for(Tuple t : tuples){
+ for(int i=0; i<t.size(); i++){
+ DataByteArray dba = (t.get(i) == null) ?
+ null : new DataByteArray(t.get(i).toString().getBytes());
+ t.set(i, dba);
+ }
+ }
+ return tuples;
+ }
+
public static File createFile(String[] data) throws Exception{
File f = File.createTempFile("tmp", "");
PrintWriter pw = new PrintWriter(f);
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld?rev=1086206&r1=1086205&r2=1086206&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld Mon Mar 28 12:24:38 2011
@@ -18,4 +18,4 @@ Reduce Plan Empty
| | |
| | Project[tuple][*] - scope-126
| |
- | |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','kmonaaafhdhcaabdgkgbhggbcohfhegjgmcoebhchcgbhjemgjhdhehiibncbnjjmhgbjnadaaabejaaaehdgjhkgfhihaaaaaaaabhhaeaaaaaaabhdhcaaeogphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccohagmgbgohdcofagihjhdgjgdgbgmfagmgbgoaaaaaaaaaaaaaaabacaaacfkaaangfgogeepggebgmgmejgohahfheemaaafgphagngbhaheaacdemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphfhegjgmcpenhfgmhegjengbhadlhihcaacfgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcfagmgbgoeaondlnechegpmlgacaaaiemaaakgneghcgpgnefgeghgfhdhbaahoaaademaaafgnelgfhjhdheaaapemgkgbhggbcphfhegjgmcpengbhadlemaaahgnemgfgbhggfhdheaabaemgkgbhggbcphfhegjgmcpemgjhdhedlemaaaegnephahdhbaahoaaafemaaaggnfcgpgphehdhbaahoaaagemaaaognfdgpggheeghcgpgnefgeghgfhdhbaahoaaademaaamgnfdgpgghefegpefgeghgfhdhbaahoaaademaaaignfegpefgeghgfhdhbaahoaaad
hihahdhcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohfhegjgmcoenhfgmhegjengbhaaaaaaaaaaaaaaaacacaaabemaaaegnengbhahbaahoaaafhihahdhcaabbgkgbhggbcohfhegjgmcoeigbhdgiengbhaafahnkmbmdbgganbadaaacegaaakgmgpgbgeeggbgdhegphcejaaajhegihcgfhdgigpgmgehihadpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhdhcaacegphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcelgfhjaaaaaaaaaaaaaaabacaaacekaaacgjgeemaaafhdgdgphagfheaabcemgkgbhggbcpgmgbgoghcpfdhehcgjgoghdlhihaaaaaaaaaaaaaaahhheaaafhdgdgphagfhdhcaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdheaaaaaaaaaaaaaaabacaaaffkaaakgphggfhcgmgpgbgegfgefkaabfhahcgpgdgfhdhdgjgoghecgbghepggfehfhagmgfhdfkaabehcgfhdhfgmhefdgjgoghgmgffehfhagmgfecgbghfkaaaehdhegbhcemaaahgdgpgmhfgngohdheaabfemgkgbhggbcphfhegjgmcpebhchcgbhjemgjhdhedlhihcaagcgphcghcogbhagbgdgigfcohagjghcogcg
bgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcoefhihahcgfhdhdgjgpgoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaadgmgpghheaacaemgphcghcpgbhagbgdgigfcpgdgpgngngpgohdcpgmgpghghgjgoghcpemgpghdlhihcaaemgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofagihjhdgjgdgbgmephagfhcgbhegphcaaaaaaaaaaaaaaabacaaamfkaaafgbgdgdhfgnfkaaangjgohahfheebhehegbgdgigfgeejaabehcgfhbhfgfhdhegfgefagbhcgbgmgmgfgmgjhdgnecaaakhcgfhdhfgmhefehjhagfemaaafgbgmgjgbhdhbaahoaaaoemaaafgjgohahfheheaablemgphcghcpgbhagbgdgigfcphagjghcpgegbhegbcpfehfhagmgfdlemaaaggjgohahfhehdhbaahoaaagemaaangmgjgogfgbghgffehcgbgdgfhcheaachemgphcghcpgbhagbgdgigfcphagjghcphagfgocphfhegjgmcpemgjgogfgbghgffehcgbgdgfhcdlemaaadgmgpghhbaahoaabeemaaahgphfhehahfhehdhbaahoaaagemaaakhagbhcgfgohefagmgbgoheaafaemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgo
gfcphagihjhdgjgdgbgmemgbhjgfhccphagmgbgohdcpfagihjhdgjgdgbgmfagmgbgodlemaaadhcgfhdheaaeeemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccpfcgfhdhfgmhedlhihcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaaegnelgfhjheaacgemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphagmgbgocpephagfhcgbhegphcelgfhjdlhihahbaahoaaapaaaappppppppdchahahahahdhcaaclgphcghcogbhagbgdgigfcogdgpgngngpgohdcogmgpghghgjgoghcogjgnhagmcoemgpghdeekemgpghghgfhccikmpnoicknfncdiacaaabemaaaegogbgngfhbaahoaaaohihaheaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdhehahahdhcaaecgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofcgfhdhfgmheaaaaaaaaaaaaaaabacaaacecaaamhcgfhehfhcgofdhegbhehfhdemaaaghcgfhdh
fgmheheaabcemgkgbhggbcpgmgbgoghcpepgcgkgfgdhedlhihaachahbaahoaaboaaaaaaaahdhbaahoaaaaaaaaaaabhhaeaaaaaaabhdhcaabbgkgbhggbcogmgbgoghcoejgohegfghgfhcbcockakephibihdiacaaabejaaafhggbgmhfgfhihcaabagkgbhggbcogmgbgoghcoeohfgngcgfhcigkmjfbnaljeoailacaaaahihaaaaaaaaahihihdhbaahoaaaaaaaaaaabhhaeaaaaaaakhbaahoaabmhihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhbaahoaabmhbaahoaaaphihdhbaahoaaaaaaaaaaaahhaeaaaaaaakhihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahiaahahi','','b','scope','true')) - scope-117
\ No newline at end of file
+ | |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','kmonaaafhdhcaabdgkgbhggbcohfhegjgmcoebhchcgbhjemgjhdhehiibncbnjjmhgbjnadaaabejaaaehdgjhkgfhihaaaaaaaabhhaeaaaaaaabhdhcaaeogphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccohagmgbgohdcofagihjhdgjgdgbgmfagmgbgoaaaaaaaaaaaaaaabacaaacfkaaangfgogeepggebgmgmejgohahfheemaaafgphagngbhaheaacdemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphfhegjgmcpenhfgmhegjengbhadlhihcaacfgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcfagmgbgoeaondlnechegpmlgacaaaiemaaakgneghcgpgnefgeghgfhdhbaahoaaademaaafgnelgfhjhdheaaapemgkgbhggbcphfhegjgmcpengbhadlemaaahgnemgfgbhggfhdheaabaemgkgbhggbcphfhegjgmcpemgjhdhedlemaaaegnephahdhbaahoaaafemaaaggnfcgpgphehdhbaahoaaagemaaaognfdgpggheeghcgpgnefgeghgfhdhbaahoaaademaaamgnfdgpgghefegpefgeghgfhdhbaahoaaademaaaignfegpefgeghgfhdhbaahoaaad
hihahdhcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohfhegjgmcoenhfgmhegjengbhaaaaaaaaaaaaaaaacacaaabemaaaegnengbhahbaahoaaafhihahdhcaabbgkgbhggbcohfhegjgmcoeigbhdgiengbhaafahnkmbmdbgganbadaaacegaaakgmgpgbgeeggbgdhegphcejaaajhegihcgfhdgigpgmgehihadpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhdhcaacegphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcelgfhjaaaaaaaaaaaaaaabacaaacekaaacgjgeemaaafhdgdgphagfheaabcemgkgbhggbcpgmgbgoghcpfdhehcgjgoghdlhihaaaaaaaaaaaaaaahhheaaafhdgdgphagfhdhcaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdheaaaaaaaaaaaaaaabacaaagfkaaaogjhdfahcgpgkgfgdhefegpefgogefkaaakgphggfhcgmgpgbgegfgefkaabfhahcgpgdgfhdhdgjgoghecgbghepggfehfhagmgfhdfkaabehcgfhdhfgmhefdgjgoghgmgffehfhagmgfecgbghejaaaihdhegbhcheedgpgmemaaahgdgpgmhfgngohdheaabfemgkgbhggbcphfhegjgmcpebhchcgbhjemgjhdhed
lhihcaagcgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcoefhihahcgfhdhdgjgpgoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaadgmgpghheaacaemgphcghcpgbhagbgdgigfcpgdgpgngngpgohdcpgmgpghghgjgoghcpemgpghdlhihcaaemgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofagihjhdgjgdgbgmephagfhcgbhegphcaaaaaaaaaaaaaaabacaaamfkaaafgbgdgdhfgnfkaaangjgohahfheebhehegbgdgigfgeejaabehcgfhbhfgfhdhegfgefagbhcgbgmgmgfgmgjhdgnecaaakhcgfhdhfgmhefehjhagfemaaafgbgmgjgbhdhbaahoaaaoemaaafgjgohahfheheaablemgphcghcpgbhagbgdgigfcphagjghcpgegbhegbcpfehfhagmgfdlemaaaggjgohahfhehdhbaahoaaagemaaangmgjgogfgbghgffehcgbgdgfhcheaachemgphcghcpgbhagbgdgigfcphagjghcphagfgocphfhegjgmcpemgjgogfgbghgffehcgbgdgfhcdlemaaadgmgpghhbaahoaabeemaaahgphfhehahfhehdhbaahoaaagemaaakhagbhcgfgohefagmgbgoheaafaemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecp
gigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccphagmgbgohdcpfagihjhdgjgdgbgmfagmgbgodlemaaadhcgfhdheaaeeemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccpfcgfhdhfgmhedlhihcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaaegnelgfhjheaacgemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphagmgbgocpephagfhcgbhegphcelgfhjdlhihahbaahoaaapaaaappppppppdchahahahahdhcaaclgphcghcogbhagbgdgigfcogdgpgngngpgohdcogmgpghghgjgoghcogjgnhagmcoemgpghdeekemgpghghgfhccikmpnoicknfncdiacaaabemaaaegogbgngfhbaahoaaaohihaheaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdhehahahdhcaaecgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofcgfhdhfgmheaaaaaaaaaaaaaaabacaaace
caaamhcgfhehfhcgofdhegbhehfhdemaaaghcgfhdhfgmheheaabcemgkgbhggbcpgmgbgoghcpepgcgkgfgdhedlhihaachahbaahoaaboaaaaaaaaaaaaaaaahdhbaahoaaaaaaaaaaabhhaeaaaaaaabhdhcaabbgkgbhggbcogmgbgoghcoejgohegfghgfhcbcockakephibihdiacaaabejaaafhggbgmhfgfhihcaabagkgbhggbcogmgbgoghcoeohfgngcgfhcigkmjfbnaljeoailacaaaahihaaaaaaaaahihihdhbaahoaaaaaaaaaaabhhaeaaaaaaakhbaahoaabmhihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhbaahoaabmhbaahoaaaphihdhbaahoaaaaaaaaaaaahhaeaaaaaaakhihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahiaahahi','','b','scope','true')) - scope-117
\ No newline at end of file
Added: pig/trunk/test/org/apache/pig/test/utils/NewLogicalPlanUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/utils/NewLogicalPlanUtil.java?rev=1086206&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/utils/NewLogicalPlanUtil.java (added)
+++ pig/trunk/test/org/apache/pig/test/utils/NewLogicalPlanUtil.java Mon Mar 28 12:24:38 2011
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test.utils;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
+import static org.junit.Assert.*;
+
+/**
+ * Additional utility functions for new logical plan
+ */
+public class NewLogicalPlanUtil {
+
+ /**
+ * Find single relational operator in plan that is instance of class c
+ * @param plan
+ * @param c
+ * @return
+ */
+ public static LogicalRelationalOperator getRelOpFromPlan(LogicalPlan plan, Class<?> c){
+ List<LogicalRelationalOperator>ops = getRelOpsFromPlan(plan,c);
+ if(ops.size() == 0)
+ return null;
+ else if(ops.size() > 1){
+ fail("more than one operator of class " + c + " found in lp " + plan);
+ return null;//never gets here
+ }else
+ return ops.get(0);
+
+ }
+
+ /**
+ * Find relational operators that are instance of class c
+ * @param plan
+ * @param c
+ * @return
+ */
+ public static List<LogicalRelationalOperator> getRelOpsFromPlan(
+ LogicalPlan plan, Class<?> c) {
+ Iterator<Operator> it = plan.getOperators();
+ ArrayList<LogicalRelationalOperator> ops =
+ new ArrayList<LogicalRelationalOperator>();
+
+ while(it.hasNext()){
+ Operator op = it.next();
+ if(op.getClass() == c){
+ ops.add((LogicalRelationalOperator) op);
+ }
+ if(op instanceof LOForEach){
+ ops.addAll(getRelOpsFromPlan(((LOForEach)op).getInnerPlan(), c));
+ }
+ }
+ return ops;
+ }
+
+}