You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2017/07/13 18:28:29 UTC
svn commit: r1801860 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
Author: knoguchi
Date: Thu Jul 13 18:28:29 2017
New Revision: 1801860
URL: http://svn.apache.org/viewvc?rev=1801860&view=rev
Log:
PIG-4767: Partition filter not pushed down when filter clause references variable from another load path (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1801860&r1=1801859&r2=1801860&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Jul 13 18:28:29 2017
@@ -36,6 +36,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4767: Partition filter not pushed down when filter clause references variable from another load path (knoguchi)
+
PIG-5270: Typo in Pig Logging (FromAlaska49 via daijy)
PIG-5263: Using wildcard doesn't work with OrcStorage (satishsaley via rohini)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java?rev=1801860&r1=1801859&r2=1801860&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java Thu Jul 13 18:28:29 2017
@@ -116,10 +116,6 @@ public class PartitionFilterOptimizer ex
return false;
loFilter = (LOFilter)succeds.get(0);
- // Filter has dependency other than load, skip optimization
- if (currentPlan.getSoftLinkPredecessors(loFilter)!=null)
- return false;
-
// we have to check more only if LoadFunc implements LoadMetada
loadFunc = loLoad.getLoadFunc();
if(!( loadFunc instanceof LoadMetadata ) ) {
Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java?rev=1801860&r1=1801859&r2=1801860&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java Thu Jul 13 18:28:29 2017
@@ -107,10 +107,6 @@ public class PredicatePushdownOptimizer
return false;
loFilter = (LOFilter) succeds.get(0);
- // Filter has dependency other than load, skip optimization
- if (currentPlan.getSoftLinkPredecessors(loFilter) != null)
- return false;
-
// we have to check more only if LoadFunc implements LoadPredicatePushdown
loadFunc = loLoad.getLoadFunc();
if (!(loadFunc instanceof LoadPredicatePushdown)) {
Modified: pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java?rev=1801860&r1=1801859&r2=1801860&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java Thu Jul 13 18:28:29 2017
@@ -61,10 +61,12 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.NotExpression;
import org.apache.pig.newplan.logical.expression.OrExpression;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
+import org.apache.pig.newplan.logical.expression.ScalarExpression;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
import org.apache.pig.newplan.logical.relational.LOFilter;
import org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.rules.LoadTypeCastInserter;
import org.apache.pig.newplan.logical.rules.PartitionFilterOptimizer;
import org.apache.pig.newplan.optimizer.PlanOptimizer;
@@ -467,10 +469,14 @@ public class TestNewPartitionFilterPushD
Operator op = newLogicalPlan.getSinks().get(0);
LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
- String actual = new PartitionFilterExtractor(null, new ArrayList<String>())
- .getExpression((LogicalExpression) filter.getFilterPlan().getSources().get(0)).toString();
- Assert.assertEquals("checking trimmed filter expression:",
- filterExpr, actual);
+ if (unsupportedExpr) {
+ String actual = getTestExpression((LogicalExpression) filter.getFilterPlan().getSources().get(0));
+ Assert.assertEquals("checking trimmed filter expression:", filterExpr, actual);
+ } else {
+ String actual = new PartitionFilterExtractor(null, new ArrayList<String>())
+ .getExpression((LogicalExpression) filter.getFilterPlan().getSources().get(0)).toString();
+ Assert.assertEquals("checking trimmed filter expression:", filterExpr, actual);
+ }
} else {
Iterator<Operator> it = newLogicalPlan.getOperators();
while( it.hasNext() ) {
@@ -707,6 +713,14 @@ public class TestNewPartitionFilterPushD
"(not (browser#'type' is null))", true);
}
+ @Test
+ public void testScalarExpressions() throws Exception {
+ String q = "z = load '1line' as (z1:int); " +
+ query3 + "b = filter a by srcid != 10 and srcid == z.z1;" +
+ "store b into 'out';";
+ testFull(q, "(srcid != 10)", "(srcid == z.z1)", true);
+ }
+
//// helper methods ///////
private PartitionFilterExtractor test(String query, List<String> partitionCols,
String expPartFilterString, String expFilterString)
@@ -917,6 +931,10 @@ public class TestNewPartitionFilterPushD
} else if (op instanceof NotExpression) {
String expr = getTestExpression(((NotExpression) op).getExpression());
return braketize("not " + expr);
+ } else if (op instanceof ScalarExpression) {
+ ScalarExpression scalar = (ScalarExpression) op;
+ return ((LogicalRelationalOperator)scalar.getImplicitReferencedOperator()).getAlias() +
+ "." + scalar.getFieldSchema().alias;
} else {
throw new FrontendException("Unsupported conversion of LogicalExpression to Expression: " + op.getName());
}