You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2018/05/18 18:43:21 UTC
[drill] 05/08: DRILL-6413: Update
ParquetFilterBuilder.visitBooleanOperator to handle simplified boolean
expression
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 5d22d708252bd5a84fe2f277f2b8b9a6abc0d9ce
Author: Arina Ielchiieva <ar...@gmail.com>
AuthorDate: Thu May 17 18:24:35 2018 +0300
DRILL-6413: Update ParquetFilterBuilder.visitBooleanOperator to handle simplified boolean expression
closes #1269
---
.../store/parquet/AbstractParquetGroupScan.java | 3 +--
.../exec/store/parquet/ParquetFilterBuilder.java | 19 +++++++++++++++----
.../store/parquet/TestParquetFilterPushDown.java | 22 +++++++++++-----------
3 files changed, 27 insertions(+), 17 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
index 1f8c535..33472bb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
@@ -255,8 +255,7 @@ public abstract class AbstractParquetGroupScan extends AbstractFileGroupScan {
logger.debug("materializedFilter : {}", ExpressionStringBuilder.toString(materializedFilter));
Set<LogicalExpression> constantBoundaries = ConstantExpressionIdentifier.getConstantExpressionSet(materializedFilter);
- filterPredicate = (ParquetFilterPredicate) ParquetFilterBuilder.buildParquetFilterPredicate(
- materializedFilter, constantBoundaries, udfUtilities);
+ filterPredicate = ParquetFilterBuilder.buildParquetFilterPredicate(materializedFilter, constantBoundaries, udfUtilities);
if (filterPredicate == null) {
return null;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
index e55425e..a8e101d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
@@ -42,6 +42,7 @@ import org.apache.drill.exec.expr.holders.ValueHolder;
import org.apache.drill.exec.expr.holders.VarDecimalHolder;
import org.apache.drill.exec.expr.stat.ParquetBooleanPredicates;
import org.apache.drill.exec.expr.stat.ParquetComparisonPredicates;
+import org.apache.drill.exec.expr.stat.ParquetFilterPredicate;
import org.apache.drill.exec.expr.stat.ParquetIsPredicates;
import org.apache.drill.exec.ops.UdfUtilities;
import org.apache.drill.exec.util.DecimalUtility;
@@ -54,7 +55,7 @@ import java.util.Set;
/**
* A visitor which visits a materialized logical expression, and build ParquetFilterPredicate
- * If a visitXXX method returns null, that means the corresponding filter branch is not qualified for pushdown.
+ * If a visitXXX method returns null, that means the corresponding filter branch is not qualified for push down.
*/
public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression, Set<LogicalExpression>, RuntimeException> {
static final Logger logger = LoggerFactory.getLogger(ParquetFilterBuilder.class);
@@ -66,12 +67,18 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression,
* @param constantBoundaries set of constant expressions
* @param udfUtilities udf utilities
*
- * @return logical expression
+ * @return parquet filter predicate
*/
- public static LogicalExpression buildParquetFilterPredicate(LogicalExpression expr, final Set<LogicalExpression> constantBoundaries, UdfUtilities udfUtilities) {
- return expr.accept(new ParquetFilterBuilder(udfUtilities), constantBoundaries);
+ public static ParquetFilterPredicate buildParquetFilterPredicate(LogicalExpression expr, final Set<LogicalExpression> constantBoundaries, UdfUtilities udfUtilities) {
+ LogicalExpression logicalExpression = expr.accept(new ParquetFilterBuilder(udfUtilities), constantBoundaries);
+ if (logicalExpression instanceof ParquetFilterPredicate) {
+ return (ParquetFilterPredicate) logicalExpression;
+ }
+ logger.debug("Logical expression {} was not qualified for filter push down", logicalExpression);
+ return null;
}
+
private ParquetFilterBuilder(UdfUtilities udfUtilities) {
this.udfUtilities = udfUtilities;
}
@@ -150,6 +157,10 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression,
return null;
}
} else {
+ if (childPredicate instanceof TypedFieldExpr) {
+ // Calcite simplifies `= true` expression to field name, wrap it with is true predicate
+ childPredicate = new ParquetIsPredicates.IsTruePredicate(childPredicate);
+ }
childPredicates.add(childPredicate);
}
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index 83a4e8e..3bbd397 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -26,6 +26,7 @@ import org.apache.drill.exec.proto.BitControl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.junit.Assert;
@@ -424,6 +425,9 @@ public class TestParquetFilterPushDown extends PlanTestBase {
final String queryNotEqualFalse = "select col_bln from dfs.`parquetFilterPush/blnTbl` where not col_bln = false";
testParquetFilterPD(queryNotEqualFalse, 4, 2, false);
+
+ final String queryEqualTrueWithAnd = "select col_bln from dfs.`parquetFilterPush/blnTbl` where col_bln = true and unk_col = 'a'";
+ testParquetFilterPD(queryEqualTrueWithAnd, 0, 2, false);
}
@Test // DRILL-5359
@@ -445,10 +449,9 @@ public class TestParquetFilterPushDown extends PlanTestBase {
public void testMultiRowGroup() throws Exception {
// multirowgroup is a parquet file with 2 rowgroups inside. One with a = 1 and the other with a = 2;
// FilterPushDown should be able to remove the rowgroup with a = 1 from the scan operator.
- final String sql = String.format("select * from dfs.`parquet/multirowgroup.parquet` where a > 1");
+ final String sql = "select * from dfs.`parquet/multirowgroup.parquet` where a > 1";
final String[] expectedPlan = {"numRowGroups=1"};
- final String[] excludedPlan = {};
- PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan);
}
//////////////////////////////////////////////////////////////////////////////////////////////////
@@ -461,23 +464,20 @@ public class TestParquetFilterPushDown extends PlanTestBase {
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=" + usedMetadataFile;
- testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern}, new String[] {});
+ testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern});
}
- private void testParquetRowGroupFilterEval(final ParquetMetadata footer, final String exprStr,
- boolean canDropExpected) throws Exception{
+ private void testParquetRowGroupFilterEval(final ParquetMetadata footer, final String exprStr, boolean canDropExpected) throws Exception{
final LogicalExpression filterExpr = parseExpr(exprStr);
testParquetRowGroupFilterEval(footer, 0, filterExpr, canDropExpected);
}
- private void testParquetRowGroupFilterEval(final ParquetMetadata footer, final int rowGroupIndex,
- final LogicalExpression filterExpr, boolean canDropExpected) throws Exception {
- boolean canDrop = ParquetRGFilterEvaluator.evalFilter(filterExpr, footer, rowGroupIndex,
- fragContext.getOptions(), fragContext);
+ private void testParquetRowGroupFilterEval(final ParquetMetadata footer, final int rowGroupIndex, final LogicalExpression filterExpr, boolean canDropExpected) {
+ boolean canDrop = ParquetRGFilterEvaluator.evalFilter(filterExpr, footer, rowGroupIndex, fragContext.getOptions(), fragContext);
Assert.assertEquals(canDropExpected, canDrop);
}
private ParquetMetadata getParquetMetaData(File file) throws IOException{
- return ParquetFileReader.readFooter(new Configuration(fs.getConf()), new Path(file.toURI()));
+ return ParquetFileReader.readFooter(new Configuration(fs.getConf()), new Path(file.toURI()), ParquetMetadataConverter.NO_FILTER);
}
}
--
To stop receiving notification emails like this one, please contact
arina@apache.org.