You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by xu...@apache.org on 2011/05/03 18:58:21 UTC
svn commit: r1099123 [10/16] - in /pig/branches/branch-0.9: ./
src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/impl/ src/org/apache/pig/impl/log...
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestPushUpFilter.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestPushUpFilter.java?rev=1099123&r1=1099122&r2=1099123&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestPushUpFilter.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestPushUpFilter.java Tue May 3 16:58:19 2011
@@ -1,1135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig.test;
-
-
-import java.util.Properties;
-
-import org.apache.pig.ExecType;
-import org.apache.pig.FilterFunc;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.logicalLayer.*;
-import org.apache.pig.impl.logicalLayer.optimizer.*;
-import org.apache.pig.test.utils.LogicalPlanTester;
-import org.apache.pig.impl.plan.optimizer.OptimizerException;
-
-import org.junit.Test;
-import org.junit.Before;
-
-/**
- * Test the logical optimizer.
- */
-
-public class TestPushUpFilter extends junit.framework.TestCase {
-
- final String FILE_BASE_LOCATION = "test/org/apache/pig/test/data/DotFiles/" ;
- static final int MAX_SIZE = 100000;
-
- PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
- LogicalPlanTester planTester = new LogicalPlanTester(pc) ;
-
- private static final String simpleEchoStreamingCommand;
- static {
- if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
- simpleEchoStreamingCommand = "perl -ne 'print \\\"$_\\\"'";
- else
- simpleEchoStreamingCommand = "perl -ne 'print \"$_\"'";
- }
-
-
- @Before
- public void tearDown() {
- planTester.reset();
- }
-
- /**
- *
- * A simple filter UDF for testing
- *
- */
- static public class MyFilterFunc extends FilterFunc {
-
- @Override
- public Boolean exec(Tuple input) {
- return false;
- }
- }
-
- @Test
- //Test to ensure that the right exception is thrown when the input list is empty
- public void testErrorEmptyInput() throws Exception {
- LogicalPlan lp = new LogicalPlan();
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
- try {
- pushUpFilter.check(lp.getRoots());
- fail("Exception Expected!");
- } catch(Exception e) {
- assertTrue(((OptimizerException)e).getErrorCode() == 2052);
- }
- }
-
- @Test
- //Test to ensure that the right exception is thrown when the input list is empty
- public void testErrorNonFilterInput() throws Exception {
- LogicalPlan lp = planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");;
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
- try {
- pushUpFilter.check(lp.getRoots());
- fail("Exception Expected!");
- } catch(Exception e) {
- assertTrue(((OptimizerException)e).getErrorCode() == 2005);
- }
- }
-
- @Test
- public void testFilterLoad() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- LogicalPlan lp = planTester.buildPlan("B = filter A by $1 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterStreaming() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = stream A through `" + simpleEchoStreamingCommand + "`;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $1 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterSort() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = order A by $1, $2;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $1 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == true);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- LOFilter filter = (LOFilter) lp.getLeaves().get(0);
- LOSort sort = (LOSort)lp.getPredecessors(filter).get(0);
-
- pushUpFilter.transform(lp.getLeaves());
-
- assertEquals(sort, lp.getLeaves().get(0));
- assertEquals(filter, lp.getPredecessors(sort).get(0));
-
- }
-
- @Test
- public void testFilterConstantConditionSort() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = order A by $1, $2;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFSort() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = order A by $1, $2;");
- LogicalPlan lp = planTester.buildPlan("D = filter B by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterDistinct() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = distinct A;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $1 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == true);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- LOFilter filter = (LOFilter) lp.getLeaves().get(0);
- LODistinct distinct = (LODistinct)lp.getPredecessors(filter).get(0);
-
- pushUpFilter.transform(lp.getLeaves());
-
- assertEquals(distinct, lp.getLeaves().get(0));
- assertEquals(filter, lp.getPredecessors(distinct).get(0));
-
- }
-
- @Test
- public void testFilterConstantConditionDistinct() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = distinct A;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFDistinct() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = distinct A;");
- LogicalPlan lp = planTester.buildPlan("D = filter B by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterForeach() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = foreach A generate $1, $2;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $0 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == true);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- LOFilter filter = (LOFilter) lp.getLeaves().get(0);
- LOForEach foreach = (LOForEach)lp.getPredecessors(filter).get(0);
-
- pushUpFilter.transform(lp.getLeaves());
-
- assertEquals(foreach, lp.getLeaves().get(0));
- assertEquals(filter, lp.getPredecessors(foreach).get(0));
-
- }
-
- @Test
- public void testFilterForeachAddedField() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = foreach A generate $1, $2, COUNT({(1)});");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $2 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterForeachCast() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = foreach A generate (chararray)$1, $2;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $0 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterCastForeach() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = foreach A generate $1, $2;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by (int)$0 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
-
- @Test
- public void testFilterConstantConditionForeach() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = foreach A generate $1, $2;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFForeach() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = foreach A generate $1, $2;");
- LogicalPlan lp = planTester.buildPlan("D = filter B by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterForeachFlatten() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = foreach A generate $1, flatten($2);");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $0 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterFilter() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = filter A by $0 != 'name';");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $1 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterSplitOutput() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("split A into B if $1 < 18, C if $1 >= 18;");
- LogicalPlan lp = planTester.buildPlan("D = filter B by $1 < 10;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterLimit() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = limit A 10;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $1 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
-
- @Test
- public void testFilterUnion() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = union A, B;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by $1 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterConstantConditionUnion() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = union A, B;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFUnion() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = union A, B;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterCross() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cross A, B;");
- planTester.buildPlan("D = filter C by $5 < 18;");
- LogicalPlan lp = planTester.buildPlan("E = limit D 10;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- LOLimit limit = (LOLimit) lp.getLeaves().get(0);
- LOFilter filter = (LOFilter) lp.getPredecessors(limit).get(0);
- LOCross cross = (LOCross)lp.getPredecessors(filter).get(0);
-
-
- assertTrue(pushUpFilter.check(lp.getPredecessors(limit)));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == true);
- assertTrue(pushUpFilter.getPushBeforeInput() == 1);
-
- pushUpFilter.transform(lp.getPredecessors(limit));
-
- assertEquals(cross, lp.getPredecessors(limit).get(0));
- assertEquals(filter, lp.getPredecessors(cross).get(1));
-
- }
-
- @Test
- public void testFilterCross1() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cross A, B;");
- planTester.buildPlan("D = filter C by $1 < 18;");
- LogicalPlan lp = planTester.buildPlan("E = limit D 10;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- LOLimit limit = (LOLimit) lp.getLeaves().get(0);
- LOFilter filter = (LOFilter) lp.getPredecessors(limit).get(0);
- LOCross cross = (LOCross)lp.getPredecessors(filter).get(0);
-
-
- assertTrue(pushUpFilter.check(lp.getPredecessors(limit)));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == true);
- assertTrue(pushUpFilter.getPushBeforeInput() == 0);
-
- pushUpFilter.transform(lp.getPredecessors(limit));
-
- assertEquals(cross, lp.getPredecessors(limit).get(0));
- assertEquals(filter, lp.getPredecessors(cross).get(0));
-
- }
-
-
- @Test
- public void testFilterCross2() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cross A, B;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by $1 < 18 and $5 < 18;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterConstantConditionCross() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cross A, B;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFCross() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cross A, B;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterCogroup() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cogroup A by $0, B by $0;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by $0 < 'name';");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterConstantConditionCogroup() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cogroup A by $0, B by $0;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFCogroup() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cogroup A by $0, B by $0;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterCogroupOuter() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cogroup A by $0, B by $0 outer;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by $0 < 'name';");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterConstantConditionCogroupOuter() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cogroup A by $0, B by $0 outer;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFCogroupOuter() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = cogroup A by $0, B by $0 outer;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterGroupBy() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = group A by $0;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $0 < 'name';");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == true);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- LOFilter filter = (LOFilter) lp.getLeaves().get(0);
- LOCogroup groupBy = (LOCogroup)lp.getPredecessors(filter).get(0);
-
- pushUpFilter.transform(lp.getLeaves());
-
- assertEquals(groupBy, lp.getLeaves().get(0));
- assertEquals(filter, lp.getPredecessors(groupBy).get(0));
- }
-
- @Test
- public void testFilterConstantConditionGroupBy() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = group A by $0;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFGroupBy() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = group A by $0;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterGroupByOuter() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = group A by $0 outer;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by $0 < 'name';");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == true);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- LOFilter filter = (LOFilter) lp.getLeaves().get(0);
- LOCogroup groupBy = (LOCogroup)lp.getPredecessors(filter).get(0);
-
- pushUpFilter.transform(lp.getLeaves());
-
- assertEquals(groupBy, lp.getLeaves().get(0));
- assertEquals(filter, lp.getPredecessors(groupBy).get(0));
-
- }
-
- @Test
- public void testFilterConstantConditionGroupByOuter() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = group A by $0 outer;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFGroupByOuter() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = group A by $0 outer;");
- LogicalPlan lp = planTester.buildPlan("C = filter B by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterFRJoin() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = join A by $0, B by $0 using \"replicated\";");
- planTester.buildPlan("D = filter C by $0 < 'name';");
- LogicalPlan lp = planTester.buildPlan("E = limit D 10;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- LOLimit limit = (LOLimit) lp.getLeaves().get(0);
- LOFilter filter = (LOFilter) lp.getPredecessors(limit).get(0);
- LOJoin frjoin = (LOJoin)lp.getPredecessors(filter).get(0);
-
- assertTrue(pushUpFilter.check(lp.getPredecessors(limit)));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == true);
- assertTrue(pushUpFilter.getPushBeforeInput() == 0);
-
-
- pushUpFilter.transform(lp.getPredecessors(limit));
-
- assertEquals(frjoin, lp.getPredecessors(limit).get(0));
- assertEquals(filter, lp.getPredecessors(frjoin).get(0));
-
- }
-
- @Test
- public void testFilterFRJoin1() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = join A by $0, B by $0 using \"replicated\";");
- planTester.buildPlan("D = filter C by $4 < 'name';");
- LogicalPlan lp = planTester.buildPlan("E = limit D 10;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- LOLimit limit = (LOLimit) lp.getLeaves().get(0);
- LOFilter filter = (LOFilter) lp.getPredecessors(limit).get(0);
- LOJoin frjoin = (LOJoin)lp.getPredecessors(filter).get(0);
-
- assertTrue(pushUpFilter.check(lp.getPredecessors(limit)));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == true);
- assertTrue(pushUpFilter.getPushBeforeInput() == 1);
-
-
- pushUpFilter.transform(lp.getPredecessors(limit));
-
- assertEquals(frjoin, lp.getPredecessors(limit).get(0));
- assertEquals(filter, lp.getPredecessors(frjoin).get(1));
-
- }
-
-
- @Test
- public void testFilterConstantConditionFRJoin() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = join A by $0, B by $0 using \"replicated\";");
- LogicalPlan lp = planTester.buildPlan("D = filter C by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFFRJoin() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = join A by $0, B by $0 using \"replicated\";");
- LogicalPlan lp = planTester.buildPlan("D = filter C by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterInnerJoin() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = join A by $0, B by $0;");
- planTester.buildPlan("D = filter C by $0 < 'name';");
- LogicalPlan lp = planTester.buildPlan("E = limit D 10;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- LOLimit limit = (LOLimit) lp.getLeaves().get(0);
- LOFilter filter = (LOFilter) lp.getPredecessors(limit).get(0);
- LOJoin join = (LOJoin)lp.getPredecessors(filter).get(0);
-
- assertTrue(pushUpFilter.check(lp.getPredecessors(limit)));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == true);
- assertTrue(pushUpFilter.getPushBeforeInput() == 0);
-
-
- pushUpFilter.transform(lp.getPredecessors(limit));
-
- assertEquals(join, lp.getPredecessors(limit).get(0));
- assertEquals(filter, lp.getPredecessors(join).get(0));
-
- }
-
- @Test
- public void testFilterInnerJoin1() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = join A by $0, B by $0;");
- planTester.buildPlan("D = filter C by $4 < 'name';");
- LogicalPlan lp = planTester.buildPlan("E = limit D 10;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- LOLimit limit = (LOLimit) lp.getLeaves().get(0);
- LOFilter filter = (LOFilter) lp.getPredecessors(limit).get(0);
- LOJoin join = (LOJoin)lp.getPredecessors(filter).get(0);
-
- assertTrue(pushUpFilter.check(lp.getPredecessors(limit)));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == true);
- assertTrue(pushUpFilter.getPushBeforeInput() == 1);
-
-
- pushUpFilter.transform(lp.getPredecessors(limit));
-
- assertEquals(join, lp.getPredecessors(limit).get(0));
- assertEquals(filter, lp.getPredecessors(join).get(1));
-
- }
-
-
- @Test
- public void testFilterConstantConditionInnerJoin() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = join A by $0, B by $0;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by 1 == 1;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterUDFInnerJoin() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name, age, preference);");
- planTester.buildPlan("C = join A by $0, B by $0;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by " + MyFilterFunc.class.getName() + "() ;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
-
- }
-
- @Test
- public void testFilterNestedForEach() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = group A by name;");
- planTester.buildPlan("C = foreach B generate A.age as age;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by age == 20;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
- }
-
- // See PIG-1289
- @Test
- public void testOutJoin() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
- planTester.buildPlan("B = load 'anotherfile' as (name);");
- planTester.buildPlan("C = join A by name LEFT OUTER, B by name;");
- LogicalPlan lp = planTester.buildPlan("D = filter C by B::name is null;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
- }
-
- // See PIG-1507
- @Test
- public void testFullOutJoin() throws Exception {
- planTester.buildPlan("A = load 'myfile' as (d1:int);");
- planTester.buildPlan("B = load 'anotherfile' as (d2:int);");
- planTester.buildPlan("c = join A by d1 full outer, B by d2;");
- LogicalPlan lp = planTester.buildPlan("d = filter c by d2 is null;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
- }
-
- // See PIG-1808
- @Test
- public void testJoinNoSchema() throws Exception {
- planTester.buildPlan("A = load '1.txt';");
- planTester.buildPlan("B = load '2.txt';");
- planTester.buildPlan("c = join A by $1 left outer, B by $1;");
- LogicalPlan lp = planTester.buildPlan("d = filter c by $2 is null;");
-
- planTester.setPlan(lp);
- planTester.setProjectionMap(lp);
-
- PushUpFilter pushUpFilter = new PushUpFilter(lp);
-
- assertTrue(!pushUpFilter.check(lp.getLeaves()));
- assertTrue(pushUpFilter.getSwap() == false);
- assertTrue(pushUpFilter.getPushBefore() == false);
- assertTrue(pushUpFilter.getPushBeforeInput() == -1);
- }
-
-}
-