[drill] branch master updated: DRILL-7761: Drill fails with OOM for the case of large filter conditions

[vo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

volodymyr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git


The following commit(s) were added to refs/heads/master by this push:
     new d8d31e5  DRILL-7761: Drill fails with OOM for the case of large filter conditions
d8d31e5 is described below

commit d8d31e5220c1343f6087797f7f7ea9b0daf7a9e3
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Tue Jun 30 22:11:50 2020 +0300

    DRILL-7761: Drill fails with OOM for the case of large filter conditions
---
 .../exec/store/parquet/FilePushDownFilter.java     |  4 +++-
 .../store/parquet/TestParquetFilterPushDown.java   | 22 ++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FilePushDownFilter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FilePushDownFilter.java
index 7721729..1fb307e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FilePushDownFilter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FilePushDownFilter.java
@@ -136,7 +136,9 @@ public abstract class FilePushDownFilter extends StoragePluginOptimizerRule {
 
     // get a conjunctions of the filter condition. For each conjunction, if it refers to ITEM or FLATTEN expression
     // then we could not pushed down. Otherwise, it's qualified to be pushed down.
-    final List<RexNode> predList = RelOptUtil.conjunctions(RexUtil.toCnf(filter.getCluster().getRexBuilder(), condition));
+    // Limits the number of nodes that can be created out of the conversion to avoid
+    // exponential growth of nodes count and further OOM
+    final List<RexNode> predList = RelOptUtil.conjunctions(RexUtil.toCnf(filter.getCluster().getRexBuilder(), 100, condition));
 
     final List<RexNode> qualifiedPredList = new ArrayList<>();
 
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index e71845e..25ba8fd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -447,6 +447,28 @@ public class TestParquetFilterPushDown extends PlanTestBase {
   }
 
   @Test
+  public void testParquetFilterPDWithLargeCondition() throws Exception {
+    test("SELECT * FROM (SELECT n.n_name AS name, n.n_nationkey AS nationkey, " +
+        "cast(n.n_regionkey AS FLOAT) AS regionkey FROM cp.`/tpch/nation.parquet` n) " +
+        "WHERE ((name = 'A' AND ((regionkey >= 0.0 AND regionkey <= 120.0 AND nationkey = 0.005))) " +
+        "OR (name = 'B' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'C' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'D' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'E' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'F' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'G' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'I' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'J' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'K' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'L' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'M' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'N' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'O' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'P' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))) " +
+        "OR (name = 'Q' AND ((regionkey >= 0.0  AND regionkey <= 120.0  AND nationkey = 0.005))))");
+  }
+
+  @Test
   public void testDatePredicateAgainstCorruptedDateCol() throws Exception {
     // Table dateTblCorrupted is created by CTAS in drill 1.8.0. Per DRILL-4203, the date column is shifted by some value.
     // The CTAS are the following, then copy to drill test resource directory.