svn commit: r1616912 - in /hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq: RelOptHiveTable.java stats/FilterSelectivityEstimator.java

[gu...@apache.org]

Author: gunther
Date: Sat Aug  9 01:29:17 2014
New Revision: 1616912

URL: http://svn.apache.org/r1616912
Log:
HIVE-7670: CBO: predicates on partition columns get applied twice in cardinality estimates. (Harish Butani via Gunther Hagleinter)

Modified:
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java?rev=1616912&r1=1616911&r2=1616912&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java Sat Aug  9 01:29:17 2014
@@ -293,4 +293,23 @@ public class RelOptHiveTable extends Rel
 
     return colStatsBldr.build();
   }
+
+  /*
+   * use to check if a set of columns are all partition columns. 
+   * true only if:
+   * - there is a prunedPartList in place
+   * - all columns in BitSet are partition
+   * columns.
+   */
+  public boolean containsPartitionColumnsOnly(BitSet cols) {
+    if (partitionList == null) {
+      return false;
+    }
+    for (int i = cols.nextSetBit(0); i >= 0; i++, i = cols.nextSetBit(i + 1)) {
+      if (!m_hivePartitionColsMap.containsKey(i)) {
+        return false;
+      }
+    }
+    return true;
+  }
 }

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java?rev=1616912&r1=1616911&r2=1616912&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java Sat Aug  9 01:29:17 2014
@@ -1,11 +1,20 @@
 package org.apache.hadoop.hive.ql.optimizer.optiq.stats;
 
+import java.util.BitSet;
+
+import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable;
+import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel;
+import org.eigenbase.rel.FilterRelBase;
+import org.eigenbase.rel.ProjectRel;
+import org.eigenbase.rel.ProjectRelBase;
 import org.eigenbase.rel.RelNode;
 import org.eigenbase.rel.metadata.RelMetadataQuery;
+import org.eigenbase.relopt.RelOptUtil;
 import org.eigenbase.relopt.RelOptUtil.InputReferencedVisitor;
 import org.eigenbase.rex.RexCall;
 import org.eigenbase.rex.RexInputRef;
 import org.eigenbase.rex.RexNode;
+import org.eigenbase.rex.RexUtil;
 import org.eigenbase.rex.RexVisitorImpl;
 import org.eigenbase.sql.SqlKind;
 
@@ -28,6 +37,15 @@ public class FilterSelectivityEstimator 
       return 1.0;
     }
 
+    /*
+     * Ignore any predicates on partition columns
+     * because we have already accounted for these in
+     * the Table row count.
+     */
+    if (isPartitionPredicate(call, m_childRel)) {
+      return 1.0;
+    }
+
     Double selectivity = null;
     SqlKind op = call.getKind();
 
@@ -178,4 +196,19 @@ public class FilterSelectivityEstimator 
 
     return maxNDV;
   }
+
+  private boolean isPartitionPredicate(RexNode expr, RelNode r) {
+    if ( r instanceof ProjectRelBase ) {
+      expr = RelOptUtil.pushFilterPastProject(expr, (ProjectRelBase) r);
+      return isPartitionPredicate(expr, ((ProjectRelBase) r).getChild());
+    } else if ( r instanceof FilterRelBase ) {
+      isPartitionPredicate(expr, ((ProjectRelBase) r).getChild());
+    } else if ( r instanceof HiveTableScanRel ) {
+      RelOptHiveTable table = (RelOptHiveTable)
+          ((HiveTableScanRel)r).getTable();
+      BitSet cols = RelOptUtil.InputFinder.bits(expr);
+      return table.containsPartitionColumnsOnly(cols);
+    }
+    return false;
+  }
 }