[GitHub] [incubator-doris] imay commented on a change in pull request #1709: Make the max recursion depth of distribution pruner configurable

[GitBox <gi...@apache.org>]

imay commented on a change in pull request #1709: Make the max recursion depth of distribution pruner configurable
URL: https://github.com/apache/incubator-doris/pull/1709#discussion_r317951999
 
 

 ##########
 File path: fe/src/main/java/org/apache/doris/planner/HashDistributionPruner.java
 ##########
 @@ -34,43 +35,55 @@
 import java.util.Map;
 import java.util.Set;
 
+/*
+ * Prune the distribution by distribution columns' predicate, recursively.
+ * It only supports binary equal predicate and in predicate with AND combination.
+ * For example:
+ *      where a = 1 and b in (2,3,4) and c in (5,6,7)
+ *      a/b/c are distribution columns
+ * 
+ * the config 'max_distribution_pruner_recursion_depth' will limit the max recursion depth of pruning.
+ * the recursion depth is calculated by the product of element number of all predicates.
+ * The above example's depth is 9(= 1 * 3 * 3)
+ * 
+ * If depth is larger than 'max_distribution_pruner_recursion_depth', all buckets will be return without pruning.
+ */
 public class HashDistributionPruner implements DistributionPruner {
     private static final Logger LOG = LogManager.getLogger(HashDistributionPruner.class);
 
     // partition list, sort by the hash code
-    private List<Long>                   partitionList;
+    private List<Long> bucketsList;
     // partition columns
-    private List<Column>                       partitionColumns;
+    private List<Column>                       distributionColumns;
     // partition column filters
-    private Map<String, PartitionColumnFilter> partitionColumnFilters;
+    private Map<String, PartitionColumnFilter> distributionColumnFilters;
     private int                                hashMod;
 
-    HashDistributionPruner(List<Long> partitions, List<Column> columns,
+    HashDistributionPruner(List<Long> bucketsList, List<Column> columns,
                            Map<String, PartitionColumnFilter> filters, int hashMod) {
-        this.partitionList = partitions;
-        this.partitionColumns = columns;
-        this.partitionColumnFilters = filters;
+        this.bucketsList = bucketsList;
+        this.distributionColumns = columns;
+        this.distributionColumnFilters = filters;
         this.hashMod = hashMod;
     }
 
     // columnId: which column to compute
     // hashKey: the key which to compute hash value
     public Collection<Long> prune(int columnId, PartitionKey hashKey, int complex) {
-        if (columnId == partitionColumns.size()) {
+        if (columnId == distributionColumns.size()) {
             // compute Hash Key
             long hashValue = hashKey.getHashValue();
-            return Lists.newArrayList(
-                    partitionList.get((int) ((hashValue & 0xffffffff) % hashMod)));
+            return Lists.newArrayList(bucketsList.get((int) ((hashValue & 0xffffffff) % hashMod)));
         }
-        Column keyColumn = partitionColumns.get(columnId);
-        PartitionColumnFilter filter = partitionColumnFilters.get(keyColumn.getName());
+        Column keyColumn = distributionColumns.get(columnId);
+        PartitionColumnFilter filter = distributionColumnFilters.get(keyColumn.getName());
         if (null == filter) {
             // no filter in this column, no partition Key
             // return all subPartition
-            return Lists.newArrayList(partitionList);
+            return Lists.newArrayList(bucketsList);
         }
         InPredicate inPredicate = filter.getInPredicate();
-        if (null == inPredicate || inPredicate.getChildren().size() * complex > 100) {
+        if (null == inPredicate || inPredicate.getChildren().size() * complex > Config.max_distribution_pruner_recursion_depth) {
 
 Review comment:
   children().size() - 1

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@doris.apache.org
For additional commands, e-mail: dev-help@doris.apache.org