You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/12/06 00:58:26 UTC

svn commit: r1548331 [1/7] - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/ ql/src/java/org/apache/hadoop/hive/ql/plan/ ql/src/java/org/apache/hadoop/hive/ql/stats/ ql/src/...

Author: hashutosh
Date: Thu Dec  5 23:58:22 2013
New Revision: 1548331

URL: http://svn.apache.org/r1548331
Log:
HIVE-5921 : Better heuristics for worst case statistics estimates for join, limit and filter operator (Prasanth J via Harish Butani)

Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_filter.q
    hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_part.q
    hive/trunk/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
    hive/trunk/ql/src/test/results/clientpositive/annotate_stats_part.q.out
    hive/trunk/ql/src/test/results/clientpositive/annotate_stats_table.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join_reordering_values.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out
    hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucket1.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucket2.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucket3.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucket4.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucket5.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucket_map_join_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucket_map_join_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketcontext_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketcontext_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketcontext_3.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketcontext_4.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketcontext_5.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketcontext_6.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketcontext_7.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketcontext_8.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin1.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin10.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin11.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin12.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin13.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin2.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin3.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin4.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin5.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin7.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin8.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin9.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out
    hive/trunk/ql/src/test/results/clientpositive/ctas_hadoop20.q.out
    hive/trunk/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out
    hive/trunk/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_map_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_6.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/input23.q.out
    hive/trunk/ql/src/test/results/clientpositive/input42.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_part1.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_part2.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_part7.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_part9.q.out
    hive/trunk/ql/src/test/results/clientpositive/join17.q.out
    hive/trunk/ql/src/test/results/clientpositive/join26.q.out
    hive/trunk/ql/src/test/results/clientpositive/join32.q.out
    hive/trunk/ql/src/test/results/clientpositive/join32_lessSize.q.out
    hive/trunk/ql/src/test/results/clientpositive/join33.q.out
    hive/trunk/ql/src/test/results/clientpositive/join34.q.out
    hive/trunk/ql/src/test/results/clientpositive/join35.q.out
    hive/trunk/ql/src/test/results/clientpositive/join9.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_filters_overlap.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_10.q.out
    hive/trunk/ql/src/test/results/clientpositive/load_dyn_part8.q.out
    hive/trunk/ql/src/test/results/clientpositive/louter_join_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/metadataonly1.q.out
    hive/trunk/ql/src/test/results/clientpositive/outer_join_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/pcr.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_join_filter.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_union_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_vc.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppr_allchildsarenull.q.out
    hive/trunk/ql/src/test/results/clientpositive/push_or.q.out
    hive/trunk/ql/src/test/results/clientpositive/rand_partitionpruner1.q.out
    hive/trunk/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out
    hive/trunk/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
    hive/trunk/ql/src/test/results/clientpositive/reduce_deduplicate.q.out
    hive/trunk/ql/src/test/results/clientpositive/regexp_extract.q.out
    hive/trunk/ql/src/test/results/clientpositive/router_join_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample1.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample2.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample4.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample5.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample6.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample7.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample8.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample9.q.out
    hive/trunk/ql/src/test/results/clientpositive/serde_user_properties.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out
    hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out
    hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out
    hive/trunk/ql/src/test/results/clientpositive/stats0.q.out
    hive/trunk/ql/src/test/results/clientpositive/stats11.q.out
    hive/trunk/ql/src/test/results/clientpositive/transform_ppr1.q.out
    hive/trunk/ql/src/test/results/clientpositive/transform_ppr2.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_explode.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_reflect.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_reflect2.q.out
    hive/trunk/ql/src/test/results/clientpositive/udtf_explode.q.out
    hive/trunk/ql/src/test/results/clientpositive/union22.q.out
    hive/trunk/ql/src/test/results/clientpositive/union24.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_ppr.q.out

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1548331&r1=1548330&r2=1548331&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Dec  5 23:58:22 2013
@@ -644,6 +644,17 @@ public class HiveConf extends Configurat
     // statistics annotation fetches column statistics for all required columns and for all
     // required partitions which can be very expensive sometimes
     HIVE_STATS_FETCH_COLUMN_STATS("hive.stats.fetch.column.stats", false),
+    // in the absence of table/partition stats, average row size will be used to
+    // estimate the number of rows/data size
+    HIVE_STATS_AVG_ROW_SIZE("hive.stats.avg.row.size", 10000),
+    // in the absence of column statistics, the estimated number of rows/data size that will
+    // emitted from join operator will depend on t factor
+    HIVE_STATS_JOIN_FACTOR("hive.stats.join.factor", (float) 1.1),
+    // in the absence of uncompressed/raw data size, total file size will be used for statistics
+    // annotation. But the file may be compressed, encoded and serialized which may be lesser in size
+    // than the actual uncompressed/raw data size. This factor will be multiplied to file size to estimate
+    // the raw data size.
+    HIVE_STATS_DESERIALIZATION_FACTOR("hive.stats.deserialization.factor", (float) 1.0),
 
     // Concurrency
     HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false),

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java?rev=1548331&r1=1548330&r2=1548331&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java Thu Dec  5 23:58:22 2013
@@ -23,7 +23,6 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 
 import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
-import org.apache.hadoop.hive.ql.exec.DemuxOperator;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.GroupByOperator;
 import org.apache.hadoop.hive.ql.exec.LimitOperator;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java?rev=1548331&r1=1548330&r2=1548331&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java Thu Dec  5 23:58:22 2013
@@ -23,12 +23,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.Stack;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
-import org.apache.hadoop.hive.ql.exec.DemuxOperator;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.GroupByOperator;
 import org.apache.hadoop.hive.ql.exec.LimitOperator;
@@ -53,7 +53,6 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.JoinDesc;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.Statistics;
-import org.apache.hadoop.hive.ql.plan.Statistics.State;
 import org.apache.hadoop.hive.ql.stats.StatsUtils;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
@@ -75,13 +74,16 @@ import com.google.common.collect.Maps;
 
 public class StatsRulesProcFactory {
 
+  private static final Log LOG = LogFactory.getLog(StatsRulesProcFactory.class.getName());
+
   /**
-   * Collect basic statistics like number of rows, data size and column level
-   * statistics from the table. Also sets the state of the available statistics.
-   * Basic and column statistics can have one of the following states
-   * COMPLETE, PARTIAL, NONE. In case of partitioned table, the basic and column
-   * stats are aggregated together to table level statistics.
-   *
+   * Collect basic statistics like number of rows, data size and column level statistics from the
+   * table. Also sets the state of the available statistics. Basic and column statistics can have
+   * one of the following states COMPLETE, PARTIAL, NONE. In case of partitioned table, the basic
+   * and column stats are aggregated together to table level statistics. Column statistics will not
+   * be collected if hive.stats.fetch.column.stats is set to false. If basic statistics is not
+   * available then number of rows will be estimated from file size and average row size (computed
+   * from schema).
    */
   public static class TableScanStatsRule extends DefaultStatsRule implements NodeProcessor {
 
@@ -102,6 +104,10 @@ public class StatsRulesProcFactory {
       Statistics stats = StatsUtils.collectStatistics(aspCtx.getConf(), partList, table, tsop);
       try {
         tsop.setStatistics(stats.clone());
+
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("[0] STATS-" + tsop.toString() + ": " + stats.extendedToString());
+        }
       } catch (CloneNotSupportedException e) {
         throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());
       }
@@ -110,23 +116,19 @@ public class StatsRulesProcFactory {
   }
 
   /**
-   * SELECT operator doesn't change the number of rows emitted from the parent
-   * operator. It changes the size of each tuple emitted. In a typical case,
-   * where only subset of columns are selected the average row size will
-   * reduce as some of the columns will be pruned. In order to accurately
-   * compute the average row size, column level statistics is required.
-   * Column level statistics stores average size of values in column which
-   * can be used to more reliably estimate the reduction in size of each
-   * tuple. In the absence of column level statistics, size of columns will be
-   * based on data type. For primitive data types size from
-   * {@link org.apache.hadoop.hive.ql.util.JavaDataModel} will be
-   * used and for variable length data types worst case will be assumed.
-   *
+   * SELECT operator doesn't change the number of rows emitted from the parent operator. It changes
+   * the size of each tuple emitted. In a typical case, where only subset of columns are selected
+   * the average row size will reduce as some of the columns will be pruned. In order to accurately
+   * compute the average row size, column level statistics is required. Column level statistics
+   * stores average size of values in column which can be used to more reliably estimate the
+   * reduction in size of each tuple. In the absence of column level statistics, size of columns
+   * will be based on data type. For primitive data types size from
+   * {@link org.apache.hadoop.hive.ql.util.JavaDataModel} will be used and for variable length data
+   * types worst case will be assumed.
    * <p>
    * <i>For more information, refer 'Estimating The Cost Of Operations' chapter in
    * "Database Systems: The Complete Book" by Garcia-Molina et. al.</i>
    * </p>
-   *
    */
   public static class SelectStatsRule extends DefaultStatsRule implements NodeProcessor {
 
@@ -155,15 +157,24 @@ public class StatsRulesProcFactory {
       try {
         if (satisfyPrecondition(parentStats)) {
           Statistics stats = parentStats.clone();
-          List<ColStatistics> colStats = StatsUtils.getColStatisticsFromExprMap(conf, parentStats,
-              sop.getColumnExprMap(), sop.getSchema());
+          List<ColStatistics> colStats =
+              StatsUtils.getColStatisticsFromExprMap(conf, parentStats, sop.getColumnExprMap(),
+                  sop.getSchema());
           long dataSize = StatsUtils.getDataSizeFromColumnStats(stats.getNumRows(), colStats);
           stats.setColumnStats(colStats);
           stats.setDataSize(dataSize);
           sop.setStatistics(stats);
+
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("[0] STATS-" + sop.toString() + ": " + stats.extendedToString());
+          }
         } else {
           if (parentStats != null) {
             sop.setStatistics(parentStats.clone());
+
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("[1] STATS-" + sop.toString() + ": " + parentStats.extendedToString());
+            }
           }
         }
       } catch (CloneNotSupportedException e) {
@@ -175,16 +186,13 @@ public class StatsRulesProcFactory {
   }
 
   /**
-   * FILTER operator does not change the average row size but it does change
-   * the number of rows emitted. The reduction in the number of rows emitted
-   * is dependent on the filter expression.
-   *
+   * FILTER operator does not change the average row size but it does change the number of rows
+   * emitted. The reduction in the number of rows emitted is dependent on the filter expression.
    * <ul>
    * <i>Notations:</i>
    * <li>T(S) - Number of tuples in relations S</li>
    * <li>V(S,A) - Number of distinct values of attribute A in relation S</li>
    * </ul>
-   *
    * <ul>
    * <i>Rules:</i> <b>
    * <li>Column equals a constant</li></b> T(S) = T(R) / V(R,A)
@@ -207,20 +215,18 @@ public class StatsRulesProcFactory {
    * <li>Multiple OR conditions</li></b> - Simple formula is to evaluate conditions independently
    * and sum the results T(S) = m1 + m2
    * <p>
-   *
    * - Alternate formula T(S) = T(R) * ( 1 - ( 1 - m1/T(R) ) * ( 1 - m2/T(R) ))
    * <p>
    * where, m1 is the number of tuples that satisfy condition1 and m2 is the number of tuples that
    * satisfy condition2
    * </ul>
    * <p>
-   * <i>Worst case:</i> If no column statistics are available, then T(R) = T(R)/2 will be
-   * used as heuristics.
+   * <i>Worst case:</i> If no column statistics are available, then evaluation of predicate
+   * expression will assume worst case (i.e; half the input rows) for each of predicate expression.
    * <p>
    * <i>For more information, refer 'Estimating The Cost Of Operations' chapter in
    * "Database Systems: The Complete Book" by Garcia-Molina et. al.</i>
    * </p>
-   *
    */
   public static class FilterStatsRule extends DefaultStatsRule implements NodeProcessor {
 
@@ -231,26 +237,47 @@ public class StatsRulesProcFactory {
       FilterOperator fop = (FilterOperator) nd;
       Operator<? extends OperatorDesc> parent = fop.getParentOperators().get(0);
       Statistics parentStats = parent.getStatistics();
+      List<String> neededCols = null;
+      if (parent instanceof TableScanOperator) {
+        TableScanOperator tsop = (TableScanOperator) parent;
+        neededCols = tsop.getNeededColumns();
+      }
 
       try {
-        if (satisfyPrecondition(parentStats)) {
+        if (parentStats != null) {
           ExprNodeDesc pred = fop.getConf().getPredicate();
 
           // evaluate filter expression and update statistics
-          long newNumRows = evaluateExpression(parentStats, pred, aspCtx);
+          long newNumRows = evaluateExpression(parentStats, pred, aspCtx, neededCols);
           Statistics st = parentStats.clone();
-          updateStats(st, newNumRows);
-          fop.setStatistics(st);
-        } else {
-          if (parentStats != null) {
 
-            // worst case, in the absence of column statistics assume half the rows are emitted
-            Statistics wcStats = getWorstCaseStats(parentStats.clone());
-            fop.setStatistics(wcStats);
+          if (satisfyPrecondition(parentStats)) {
+
+            // update statistics based on column statistics.
+            // OR conditions keeps adding the stats independently, this may
+            // result in number of rows getting more than the input rows in
+            // which case stats need not be updated
+            if (newNumRows <= parentStats.getNumRows()) {
+              updateStats(st, newNumRows, true);
+            }
+
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("[0] STATS-" + fop.toString() + ": " + st.extendedToString());
+            }
+          } else {
+
+            // update only the basic statistics in the absence of column statistics
+            if (newNumRows <= parentStats.getNumRows()) {
+              updateStats(st, newNumRows, false);
+            }
+
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("[1] STATS-" + fop.toString() + ": " + st.extendedToString());
+            }
           }
+          fop.setStatistics(st);
+          aspCtx.setAndExprStats(null);
         }
-
-        aspCtx.setAndExprStats(null);
       } catch (CloneNotSupportedException e) {
         throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());
       }
@@ -258,7 +285,7 @@ public class StatsRulesProcFactory {
     }
 
     private long evaluateExpression(Statistics stats, ExprNodeDesc pred,
-        AnnotateStatsProcCtx aspCtx) throws CloneNotSupportedException {
+        AnnotateStatsProcCtx aspCtx, List<String> neededCols) throws CloneNotSupportedException {
       long newNumRows = 0;
       Statistics andStats = null;
       if (pred instanceof ExprNodeGenericFuncDesc) {
@@ -272,28 +299,24 @@ public class StatsRulesProcFactory {
 
           // evaluate children
           for (ExprNodeDesc child : genFunc.getChildren()) {
-            newNumRows = evaluateChildExpr(aspCtx.getAndExprStats(), child, aspCtx);
-            updateStats(aspCtx.getAndExprStats(), newNumRows);
+            newNumRows = evaluateChildExpr(aspCtx.getAndExprStats(), child, aspCtx, neededCols);
+            if (satisfyPrecondition(aspCtx.getAndExprStats())) {
+              updateStats(aspCtx.getAndExprStats(), newNumRows, true);
+            } else {
+              updateStats(aspCtx.getAndExprStats(), newNumRows, false);
+            }
           }
-        } else {
-
+        } else if (udf instanceof GenericUDFOPOr) {
           // for OR condition independently compute and update stats
-          if (udf instanceof GenericUDFOPOr) {
-            for (ExprNodeDesc child : genFunc.getChildren()) {
-              newNumRows += evaluateChildExpr(stats, child, aspCtx);
-            }
-          } else if (udf instanceof GenericUDFOPNot) {
-            newNumRows = evaluateNotExpr(stats, pred, aspCtx);
-          } else if (udf instanceof GenericUDFOPNotNull) {
-            newNumRows = evaluateColEqualsNullExpr(stats, pred, aspCtx);
-            newNumRows = stats.getNumRows() - newNumRows;
-          } else if (udf instanceof GenericUDFOPNull) {
-            newNumRows = evaluateColEqualsNullExpr(stats, pred, aspCtx);
-          } else {
-
-            // single predicate condition
-            newNumRows = evaluateChildExpr(stats, pred, aspCtx);
+          for (ExprNodeDesc child : genFunc.getChildren()) {
+            newNumRows += evaluateChildExpr(stats, child, aspCtx, neededCols);
           }
+        } else if (udf instanceof GenericUDFOPNot) {
+          newNumRows = evaluateNotExpr(stats, pred, aspCtx, neededCols);
+        } else {
+
+          // single predicate condition
+          newNumRows = evaluateChildExpr(stats, pred, aspCtx, neededCols);
         }
       } else if (pred instanceof ExprNodeColumnDesc) {
 
@@ -304,19 +327,20 @@ public class StatsRulesProcFactory {
         String colType = encd.getTypeString();
         if (colType.equalsIgnoreCase(serdeConstants.BOOLEAN_TYPE_NAME)) {
           ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);
-          return cs.getNumTrues();
-        } else {
-
-          // if not boolean column return half the number of rows
-          return stats.getNumRows() / 2;
+          if (cs != null) {
+            return cs.getNumTrues();
+          }
         }
+
+        // if not boolean column return half the number of rows
+        return stats.getNumRows() / 2;
       }
 
       return newNumRows;
     }
 
-    private long evaluateNotExpr(Statistics stats, ExprNodeDesc pred, AnnotateStatsProcCtx aspCtx)
-        throws CloneNotSupportedException {
+    private long evaluateNotExpr(Statistics stats, ExprNodeDesc pred, AnnotateStatsProcCtx aspCtx,
+        List<String> neededCols) throws CloneNotSupportedException {
 
       long numRows = stats.getNumRows();
 
@@ -329,7 +353,7 @@ public class StatsRulesProcFactory {
             // GenericUDF
             long newNumRows = 0;
             for (ExprNodeDesc child : ((ExprNodeGenericFuncDesc) pred).getChildren()) {
-              newNumRows = evaluateChildExpr(stats, child, aspCtx);
+              newNumRows = evaluateChildExpr(stats, child, aspCtx, neededCols);
             }
             return numRows - newNumRows;
           } else if (leaf instanceof ExprNodeConstantDesc) {
@@ -348,18 +372,18 @@ public class StatsRulesProcFactory {
             String colType = encd.getTypeString();
             if (colType.equalsIgnoreCase(serdeConstants.BOOLEAN_TYPE_NAME)) {
               ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);
-              return cs.getNumFalses();
-            } else {
-
-              // if not boolean column return half the number of rows
-              return numRows / 2;
+              if (cs != null) {
+                return cs.getNumFalses();
+              }
             }
+            // if not boolean column return half the number of rows
+            return numRows / 2;
           }
         }
       }
 
       // worst case
-      return numRows;
+      return numRows / 2;
     }
 
     private long evaluateColEqualsNullExpr(Statistics stats, ExprNodeDesc pred,
@@ -380,26 +404,19 @@ public class StatsRulesProcFactory {
             ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);
             if (cs != null) {
               long dvs = cs.getCountDistint();
-              // if NULLs exists, add 1 to distinct count
-              if (cs.getNumNulls() > 0) {
-                dvs += 1;
-              }
-              if (dvs != 0) {
-                return numRows / dvs;
-              } else {
-                return numRows;
-              }
+              numRows = dvs == 0 ? numRows / 2 : numRows / dvs;
+              return numRows;
             }
           }
         }
       }
 
       // worst case
-      return numRows;
+      return numRows / 2;
     }
 
-    private long evaluateChildExpr(Statistics stats, ExprNodeDesc child, AnnotateStatsProcCtx aspCtx)
-        throws CloneNotSupportedException {
+    private long evaluateChildExpr(Statistics stats, ExprNodeDesc child,
+        AnnotateStatsProcCtx aspCtx, List<String> neededCols) throws CloneNotSupportedException {
 
       long numRows = stats.getNumRows();
 
@@ -421,19 +438,19 @@ public class StatsRulesProcFactory {
                 isConst = true;
                 continue;
               }
+
+              // if column name is not contained in needed column list then it
+              // is a partition column. We do not need to evaluate partition columns
+              // in filter expression since it will be taken care by partitio pruner
+              if (neededCols != null && !neededCols.contains(colName)) {
+                return numRows;
+              }
+
               ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);
               if (cs != null) {
                 long dvs = cs.getCountDistint();
-                // if NULLs exists, add 1 to distinct count
-                if (cs.getNumNulls() > 0) {
-                  dvs += 1;
-                }
-
-                if (dvs != 0) {
-                  return numRows / dvs;
-                } else {
-                  return numRows;
-                }
+                numRows = dvs == 0 ? numRows / 2 : numRows / dvs;
+                return numRows;
               }
             } else if (leaf instanceof ExprNodeColumnDesc) {
               ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc) leaf;
@@ -442,53 +459,56 @@ public class StatsRulesProcFactory {
 
               // if const is first argument then evaluate the result
               if (isConst) {
+
+                // if column name is not contained in needed column list then it
+                // is a partition column. We do not need to evaluate partition columns
+                // in filter expression since it will be taken care by partitio pruner
+                if (neededCols != null && neededCols.indexOf(colName) == -1) {
+                  return numRows;
+                }
+
                 ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName);
                 if (cs != null) {
                   long dvs = cs.getCountDistint();
-                  // if NULLs exists, add 1 to distinct count
-                  if (cs.getNumNulls() > 0) {
-                    dvs += 1;
-                  }
-
-                  if (dvs != 0) {
-                    return numRows / dvs;
-                  } else {
-                    return numRows;
-                  }
+                  numRows = dvs == 0 ? numRows / 2 : numRows / dvs;
+                  return numRows;
                 }
               }
             }
           }
         } else if (udf instanceof GenericUDFOPNotEqual) {
           return numRows;
-        } else if (udf instanceof GenericUDFOPEqualOrGreaterThan ||
-            udf instanceof GenericUDFOPEqualOrLessThan ||
-            udf instanceof GenericUDFOPGreaterThan ||
-            udf instanceof GenericUDFOPLessThan) {
+        } else if (udf instanceof GenericUDFOPEqualOrGreaterThan
+            || udf instanceof GenericUDFOPEqualOrLessThan || udf instanceof GenericUDFOPGreaterThan
+            || udf instanceof GenericUDFOPLessThan) {
           return numRows / 3;
-        } else {
-          return evaluateExpression(stats, genFunc, aspCtx);
+        } else if (udf instanceof GenericUDFOPNotNull) {
+          long newNumRows = evaluateColEqualsNullExpr(stats, genFunc, aspCtx);
+          return stats.getNumRows() - newNumRows;
+        } else if (udf instanceof GenericUDFOPNull) {
+          return evaluateColEqualsNullExpr(stats, genFunc, aspCtx);
+        } else if (udf instanceof GenericUDFOPAnd || udf instanceof GenericUDFOPOr
+            || udf instanceof GenericUDFOPNot) {
+          return evaluateExpression(stats, genFunc, aspCtx, neededCols);
         }
       }
 
       // worst case
-      return numRows;
+      return numRows / 2;
     }
 
   }
 
   /**
-   * GROUPBY operator changes the number of rows. The number of rows emitted
-   * by GBY operator will be atleast 1 or utmost T(R) (number of rows in relation T)
-   * based on the aggregation. A better estimate can be found if we have column statistics
-   * on the columns that we are grouping on.
+   * GROUPBY operator changes the number of rows. The number of rows emitted by GBY operator will be
+   * atleast 1 or utmost T(R) (number of rows in relation T) based on the aggregation. A better
+   * estimate can be found if we have column statistics on the columns that we are grouping on.
    * <p>
    * Suppose if we are grouping by attributes A,B,C and if statistics for columns A,B,C are
    * available then a better estimate can be found by taking the smaller of product of V(R,[A,B,C])
    * (product of distinct cardinalities of A,B,C) and T(R)/2.
    * <p>
    * T(R) = min (T(R)/2 , V(R,[A,B,C]) ---> [1]
-   *
    * <p>
    * In the presence of grouping sets, map-side GBY will emit more rows depending on the size of
    * grouping set (input rows * size of grouping set). These rows will get reduced because of
@@ -503,27 +523,23 @@ public class StatsRulesProcFactory {
    * T(R) = min(T(R)/2, T(R, GBY(A,B)) + T(R, GBY(A)) + T(R, GBY(B)) + 1))
    * <p>
    * where, GBY(A,B), GBY(B), GBY(B) are the GBY rules mentioned above [1]
-   *
    * <p>
    * If <b>hash-aggregation is disabled</b>, apply the GBY rule [1] and then multiply the result by
    * number of elements in grouping set T(R) = T(R) * length_of_grouping_set. Since we do not know
    * if hash-aggregation is enabled or disabled during compile time, we will assume worst-case i.e,
    * hash-aggregation is disabled
-   *
    * <p>
    * NOTE: The number of rows from map-side GBY operator is dependent on map-side parallelism i.e,
    * number of mappers. The map-side parallelism is expected from hive config
    * "hive.stats.map.parallelism". If the config is not set then default parallelism of 1 will be
    * assumed.
-   *
    * <p>
-   * <i>Worst case:</i> If no column statistics are available, then T(R) = T(R)/2 will be
-   * used as heuristics.
+   * <i>Worst case:</i> If no column statistics are available, then T(R) = T(R)/2 will be used as
+   * heuristics.
    * <p>
    * <i>For more information, refer 'Estimating The Cost Of Operations' chapter in
    * "Database Systems: The Complete Book" by Garcia-Molina et. al.</i>
    * </p>
-   *
    */
   public static class GroupByStatsRule extends DefaultStatsRule implements NodeProcessor {
 
@@ -535,8 +551,8 @@ public class StatsRulesProcFactory {
       Statistics parentStats = parent.getStatistics();
       AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;
       HiveConf conf = aspCtx.getConf();
-      int mapSideParallelism = HiveConf.getIntVar(conf,
-          HiveConf.ConfVars.HIVE_STATS_MAP_SIDE_PARALLELISM);
+      int mapSideParallelism =
+          HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_MAP_SIDE_PARALLELISM);
       List<AggregationDesc> aggDesc = gop.getConf().getAggregators();
       Map<String, ExprNodeDesc> colExprMap = gop.getColumnExprMap();
       RowSchema rs = gop.getSchema();
@@ -546,8 +562,8 @@ public class StatsRulesProcFactory {
         if (satisfyPrecondition(parentStats)) {
           stats = parentStats.clone();
 
-          List<ColStatistics> colStats = StatsUtils.getColStatisticsFromExprMap(conf, parentStats,
-              colExprMap, rs);
+          List<ColStatistics> colStats =
+              StatsUtils.getColStatisticsFromExprMap(conf, parentStats, colExprMap, rs);
           stats.setColumnStats(colStats);
           long dvProd = 1;
           long newNumRows = 0;
@@ -563,7 +579,8 @@ public class StatsRulesProcFactory {
             } else {
 
               // partial column statistics on grouping attributes case.
-              // if column statistics on grouping attribute is missing, then assume worst case.
+              // if column statistics on grouping attribute is missing, then
+              // assume worst case.
               // GBY rule will emit half the number of rows if dvProd is 0
               dvProd = 0;
               break;
@@ -574,8 +591,8 @@ public class StatsRulesProcFactory {
           if (gop.getChildOperators().get(0) instanceof ReduceSinkOperator) {
 
             // since we do not know if hash-aggregation will be enabled or disabled
-            // at runtime we will assume that map-side group by does not do any reduction.
-            // hence no group by rule will be applied
+            // at runtime we will assume that map-side group by does not do any
+            // reduction.hence no group by rule will be applied
 
             // map-side grouping set present. if grouping set is present then
             // multiply the number of rows by number of elements in grouping set
@@ -599,13 +616,13 @@ public class StatsRulesProcFactory {
 
               // map side no grouping set
               newNumRows = stats.getNumRows() * mapSideParallelism;
-              updateStats(stats, newNumRows);
+              updateStats(stats, newNumRows, true);
             }
           } else {
 
             // reduce side
             newNumRows = applyGBYRule(stats.getNumRows(), dvProd);
-            updateStats(stats, newNumRows);
+            updateStats(stats, newNumRows, true);
           }
         } else {
           if (parentStats != null) {
@@ -618,7 +635,9 @@ public class StatsRulesProcFactory {
             } else {
 
               // reduce side
-              stats = getWorstCaseStats(parentStats);
+              stats = parentStats.clone();
+              long newNumRows = parentStats.getNumRows() / 2;
+              updateStats(stats, newNumRows, false);
             }
           }
         }
@@ -647,14 +666,19 @@ public class StatsRulesProcFactory {
           stats.addToColumnStats(aggColStats);
 
           // if UDAF present and if column expression map is empty then it must
-          // be full aggregation query like count(*) in which case number of rows will be 1
+          // be full aggregation query like count(*) in which case number of
+          // rows will be 1
           if (colExprMap.isEmpty()) {
             stats.setNumRows(1);
-            updateStats(stats, 1);
+            updateStats(stats, 1, true);
           }
         }
 
         gop.setStatistics(stats);
+
+        if (LOG.isDebugEnabled() && stats != null) {
+          LOG.debug("[0] STATS-" + gop.toString() + ": " + stats.extendedToString());
+        }
       } catch (CloneNotSupportedException e) {
         throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());
       }
@@ -683,7 +707,6 @@ public class StatsRulesProcFactory {
    * will have a tuple in R T(RXS) = T(S) (we need histograms for this)</li> <li>Both R & S relation
    * have same value for join-key. Ex: bool column with all true values T(RXS) = T(R) * T(S) (we
    * need histograms for this. counDistinct = 1 and same value)</li>
-   *
    * <p>
    * In the absence of histograms, we can use the following general case
    * <p>
@@ -695,10 +718,11 @@ public class StatsRulesProcFactory {
    * <p>
    * T(RXS) = T(R)*T(S)/max(V(R,y1), V(S,y1)) * max(V(R,y2), V(S,y2)), where y1 and y2 are the join
    * attributes
-   *
    * <p>
-   * <i>Worst case:</i> If no column statistics are available, then T(RXS) = T(R)*T(S)/2 will be
-   * used as heuristics.
+   * <i>Worst case:</i> If no column statistics are available, then T(RXS) = joinFactor * max(T(R),
+   * T(S)) * (numParents - 1) will be used as heuristics. joinFactor is from hive.stats.join.factor
+   * hive config. In the worst case, since we do not know any information about join keys (and hence
+   * which of the 3 cases to use), we let it to the user to provide the join factor.
    * <p>
    * <i>For more information, refer 'Estimating The Cost Of Operations' chapter in
    * "Database Systems: The Complete Book" by Garcia-Molina et. al.</i>
@@ -730,16 +754,15 @@ public class StatsRulesProcFactory {
           }
         }
 
-        try {
         if (allSatisfyPreCondition) {
 
-          // statistics object that is combination of statistics from all relations involved in JOIN
+          // statistics object that is combination of statistics from all
+          // relations involved in JOIN
           Statistics stats = new Statistics();
           long prodRows = 1;
           List<Long> distinctVals = Lists.newArrayList();
           boolean multiAttr = false;
 
-
           Map<String, ColStatistics> joinedColStats = Maps.newHashMap();
           Map<Integer, List<String>> joinKeys = Maps.newHashMap();
 
@@ -756,33 +779,34 @@ public class StatsRulesProcFactory {
               multiAttr = true;
             }
 
-            // compute fully qualified join key column names. this name will be used to
-            // quickly look-up for column statistics of join key.
-            // TODO: expressions in join condition will be ignored. assign internal name
-            // for expressions and estimate column statistics for expression.
-            List<String> fqCols = StatsUtils.getFullQualifedColNameFromExprs(keyExprs,
-                parent.getColumnExprMap());
+            // compute fully qualified join key column names. this name will be
+            // used to quickly look-up for column statistics of join key.
+            // TODO: expressions in join condition will be ignored. assign
+            // internal name for expressions and estimate column statistics for expression.
+            List<String> fqCols =
+                StatsUtils.getFullQualifedColNameFromExprs(keyExprs, parent.getColumnExprMap());
             joinKeys.put(pos, fqCols);
 
             Map<String, ExprNodeDesc> colExprMap = parent.getColumnExprMap();
             RowSchema rs = parent.getSchema();
 
             // get column statistics for all output columns
-            List<ColStatistics> cs = StatsUtils.getColStatisticsFromExprMap(conf, parentStats,
-                colExprMap, rs);
+            List<ColStatistics> cs =
+                StatsUtils.getColStatisticsFromExprMap(conf, parentStats, colExprMap, rs);
             for (ColStatistics c : cs) {
               if (c != null) {
                 joinedColStats.put(c.getFullyQualifiedColName(), c);
               }
             }
 
-            // since new statistics is derived from all relations involved in JOIN,
-            // we need to update the state information accordingly
+            // since new statistics is derived from all relations involved in
+            // JOIN, we need to update the state information accordingly
             stats.updateColumnStatsState(parentStats.getColumnStatsState());
           }
 
-          // compute denominator i.e, max(V(R,Y), V(S,Y)) in case of single attribute join.
-          // else max(V(R,y1), V(S,y1)) * max(V(R,y2), V(S,y2)) in case of multi-attribute join
+          // compute denominator i.e, max(V(R,Y), V(S,Y)) in case of single
+          // attribute join, else max(V(R,y1), V(S,y1)) * max(V(R,y2), V(S,y2))
+          // in case of multi-attribute join
           long denom = 1;
           if (multiAttr) {
             List<Long> perAttrDVs = Lists.newArrayList();
@@ -845,28 +869,45 @@ public class StatsRulesProcFactory {
           stats.setNumRows(newRowCount);
           stats.setDataSize(StatsUtils.getDataSizeFromColumnStats(newRowCount, outColStats));
           jop.setStatistics(stats);
-          } else {
 
-            // worst case, when no column statistics are available
-            if (parents.size() > 1) {
-              Statistics wcStats = new Statistics();
-              Statistics stp1 = parents.get(0).getStatistics();
-              long numRows = stp1.getNumRows();
-              long avgRowSize = stp1.getAvgRowSize();
-              for (int i = 1; i < parents.size(); i++) {
-                stp1 = parents.get(i).getStatistics();
-                numRows = (numRows * stp1.getNumRows()) / 2;
-                avgRowSize += stp1.getAvgRowSize();
-              }
-              wcStats.setNumRows(numRows);
-              wcStats.setDataSize(numRows * avgRowSize);
-              jop.setStatistics(wcStats);
-            } else {
-              jop.setStatistics(parents.get(0).getStatistics().clone());
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("[0] STATS-" + jop.toString() + ": " + stats.extendedToString());
+          }
+        } else {
+
+          // worst case when there are no column statistics
+          float joinFactor = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVE_STATS_JOIN_FACTOR);
+          int numParents = parents.size();
+          List<Long> parentRows = Lists.newArrayList();
+          List<Long> parentSizes = Lists.newArrayList();
+          int maxRowIdx = 0;
+          long maxRowCount = 0;
+          int idx = 0;
+
+          for (Operator<? extends OperatorDesc> op : parents) {
+            Statistics ps = op.getStatistics();
+            long rowCount = ps.getNumRows();
+            if (rowCount > maxRowCount) {
+              maxRowCount = rowCount;
+              maxRowIdx = idx;
             }
+            parentRows.add(rowCount);
+            parentSizes.add(ps.getDataSize());
+            idx++;
+          }
+
+          long maxDataSize = parentSizes.get(maxRowIdx);
+          long newNumRows = (long) (joinFactor * maxRowCount * (numParents - 1));
+          long newDataSize = (long) (joinFactor * maxDataSize * (numParents - 1));
+
+          Statistics wcStats = new Statistics();
+          wcStats.setNumRows(newNumRows);
+          wcStats.setDataSize(newDataSize);
+          jop.setStatistics(wcStats);
+
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("[1] STATS-" + jop.toString() + ": " + wcStats.extendedToString());
           }
-        } catch (CloneNotSupportedException e) {
-          throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());
         }
       }
       return null;
@@ -874,22 +915,22 @@ public class StatsRulesProcFactory {
 
     private long getDenominator(List<Long> distinctVals) {
 
-      if(distinctVals.isEmpty()) {
+      if (distinctVals.isEmpty()) {
 
-        // TODO: in union20.q the tab alias is not properly propagated down the operator
-        // tree. This happens when UNION ALL is used as sub query. Hence, even if column
-        // statistics are available, the tab alias will be null which will fail to get
-        // proper column statistics. For now assume, worst case in which denominator is 2.
+        // TODO: in union20.q the tab alias is not properly propagated down the
+        // operator tree. This happens when UNION ALL is used as sub query. Hence, even
+        // if column statistics are available, the tab alias will be null which will fail
+        // to get proper column statistics. For now assume, worst case in which
+        // denominator is 2.
         return 2;
       }
 
-      // simple join from 2 relations
-      // denom = max(v1, v2)
+      // simple join from 2 relations: denom = max(v1, v2)
       if (distinctVals.size() <= 2) {
         return Collections.max(distinctVals);
       } else {
 
-        // join from multiple relations
+        // join from multiple relations:
         // denom = max(v1, v2) * max(v2, v3) * max(v3, v4)
         long denom = 1;
         for (int i = 0; i < distinctVals.size() - 1; i++) {
@@ -909,7 +950,6 @@ public class StatsRulesProcFactory {
 
   /**
    * LIMIT operator changes the number of rows and thereby the data size.
-   *
    */
   public static class LimitStatsRule extends DefaultStatsRule implements NodeProcessor {
 
@@ -919,6 +959,8 @@ public class StatsRulesProcFactory {
       LimitOperator lop = (LimitOperator) nd;
       Operator<? extends OperatorDesc> parent = lop.getParentOperators().get(0);
       Statistics parentStats = parent.getStatistics();
+      AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx;
+      HiveConf conf = aspCtx.getConf();
 
       try {
         long limit = -1;
@@ -927,25 +969,37 @@ public class StatsRulesProcFactory {
         if (satisfyPrecondition(parentStats)) {
           Statistics stats = parentStats.clone();
 
-          // if limit is greater than available rows then do not update statistics
+          // if limit is greater than available rows then do not update
+          // statistics
           if (limit <= parentStats.getNumRows()) {
-            updateStats(stats, limit);
+            updateStats(stats, limit, true);
           }
           lop.setStatistics(stats);
+
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("[0] STATS-" + lop.toString() + ": " + stats.extendedToString());
+          }
         } else {
           if (parentStats != null) {
 
-            // in the absence of column statistics, compute data size based on based
-            // on average row size
+            // in the absence of column statistics, compute data size based on
+            // based on average row size
             Statistics wcStats = parentStats.clone();
             if (limit <= parentStats.getNumRows()) {
               long numRows = limit;
               long avgRowSize = parentStats.getAvgRowSize();
+              if (avgRowSize <= 0) {
+                avgRowSize = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_AVG_ROW_SIZE);
+              }
               long dataSize = avgRowSize * limit;
               wcStats.setNumRows(numRows);
               wcStats.setDataSize(dataSize);
             }
             lop.setStatistics(wcStats);
+
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("[1] STATS-" + lop.toString() + ": " + wcStats.extendedToString());
+            }
           }
         }
       } catch (CloneNotSupportedException e) {
@@ -958,7 +1012,6 @@ public class StatsRulesProcFactory {
 
   /**
    * Default rule is to aggregate the statistics from all its parent operators.
-   *
    */
   public static class DefaultStatsRule implements NodeProcessor {
 
@@ -973,8 +1026,8 @@ public class StatsRulesProcFactory {
         if (stats == null) {
           if (op.getParentOperators() != null) {
 
-            // if parent statistics is null then that branch of the tree is not walked yet.
-            // don't update the stats until all branches are walked
+            // if parent statistics is null then that branch of the tree is not
+            // walked yet. don't update the stats until all branches are walked
             if (isAllParentsContainStatistics(op)) {
               stats = new Statistics();
               for (Operator<? extends OperatorDesc> parent : op.getParentOperators()) {
@@ -985,6 +1038,10 @@ public class StatsRulesProcFactory {
                   stats.updateColumnStatsState(parentStats.getColumnStatsState());
                   stats.addToColumnStats(parentStats.getColumnStats());
                   op.getConf().setStatistics(stats);
+
+                  if (LOG.isDebugEnabled()) {
+                    LOG.debug("[0] STATS-" + op.toString() + ": " + stats.extendedToString());
+                  }
                 }
               }
             }
@@ -1036,37 +1093,43 @@ public class StatsRulesProcFactory {
 
   /**
    * Update the basic statistics of the statistics object based on the row number
-   *
    * @param stats
    *          - statistics to be updated
    * @param newNumRows
    *          - new number of rows
+   * @param useColStats
+   *          - use column statistics to compute data size
    */
-  static void updateStats(Statistics stats, long newNumRows) {
+  static void updateStats(Statistics stats, long newNumRows, boolean useColStats) {
     long oldRowCount = stats.getNumRows();
     double ratio = (double) newNumRows / (double) oldRowCount;
     stats.setNumRows(newNumRows);
 
-    List<ColStatistics> colStats = stats.getColumnStats();
-    for (ColStatistics cs : colStats) {
-      long oldNumNulls = cs.getNumNulls();
-      long oldDV = cs.getCountDistint();
-      long newNumNulls = Math.round(ratio * oldNumNulls);
-      long newDV = oldDV;
-
-      // if ratio is greater than 1, then number of rows increases. This can happen
-      // when some operators like GROUPBY duplicates the input rows in which case
-      // number of distincts should not change. Update the distinct count only when
-      // the output number of rows is less than input number of rows.
-      if (ratio <= 1.0) {
-        newDV = Math.round(ratio * oldDV);
+    if (useColStats) {
+      List<ColStatistics> colStats = stats.getColumnStats();
+      for (ColStatistics cs : colStats) {
+        long oldNumNulls = cs.getNumNulls();
+        long oldDV = cs.getCountDistint();
+        long newNumNulls = Math.round(ratio * oldNumNulls);
+        long newDV = oldDV;
+
+        // if ratio is greater than 1, then number of rows increases. This can happen
+        // when some operators like GROUPBY duplicates the input rows in which case
+        // number of distincts should not change. Update the distinct count only when
+        // the output number of rows is less than input number of rows.
+        if (ratio <= 1.0) {
+          newDV = Math.round(ratio * oldDV);
+        }
+        cs.setNumNulls(newNumNulls);
+        cs.setCountDistint(newDV);
       }
-      cs.setNumNulls(newNumNulls);
-      cs.setCountDistint(newDV);
+      stats.setColumnStats(colStats);
+      long newDataSize = StatsUtils.getDataSizeFromColumnStats(newNumRows, colStats);
+      stats.setDataSize(newDataSize);
+    } else {
+      long newDataSize = (long) (ratio * stats.getDataSize());
+      stats.setDataSize(newDataSize);
     }
-    stats.setColumnStats(colStats);
-    long newDataSize = StatsUtils.getDataSizeFromColumnStats(newNumRows, colStats);
-    stats.setDataSize(newDataSize);
   }
 
   static boolean satisfyPrecondition(Statistics stats) {
@@ -1074,16 +1137,4 @@ public class StatsRulesProcFactory {
         && !stats.getColumnStatsState().equals(Statistics.State.NONE);
   }
 
-  static Statistics getWorstCaseStats(Statistics stats) throws CloneNotSupportedException {
-    Statistics wcClone = stats.clone();
-    long numRows = wcClone.getNumRows() / 2;
-    long dataSize = wcClone.getDataSize() / 2;
-    long avgRowSize = wcClone.getAvgRowSize();
-    if (numRows > 0) {
-      dataSize = avgRowSize * numRows;
-    }
-    wcClone.setNumRows(numRows);
-    wcClone.setDataSize(dataSize);
-    return wcClone;
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java?rev=1548331&r1=1548330&r2=1548331&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java Thu Dec  5 23:58:22 2013
@@ -115,6 +115,21 @@ public class Statistics implements Seria
     return sb.toString();
   }
 
+  public String extendedToString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(" numRows: ");
+    sb.append(numRows);
+    sb.append(" dataSize: ");
+    sb.append(dataSize);
+    sb.append(" basicStatsState: ");
+    sb.append(basicStatsState);
+    sb.append(" colStatsState: ");
+    sb.append(columnStatsState);
+    sb.append(" colStats: ");
+    sb.append(columnStats);
+    return sb.toString();
+  }
+
   @Override
   public Statistics clone() throws CloneNotSupportedException {
     Statistics clone = new Statistics(numRows, dataSize);
@@ -215,7 +230,10 @@ public class Statistics implements Seria
   }
 
   public ColStatistics getColumnStatisticsFromFQColName(String fqColName) {
-    return columnStats.get(fqColName);
+    if (columnStats != null) {
+      return columnStats.get(fqColName);
+    }
+    return null;
   }
 
   public ColStatistics getColumnStatisticsFromColName(String colName) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java?rev=1548331&r1=1548330&r2=1548331&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java Thu Dec  5 23:58:22 2013
@@ -3,6 +3,8 @@ package org.apache.hadoop.hive.ql.stats;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.StatsSetupConst;
@@ -62,9 +64,10 @@ import com.google.common.collect.Lists;
 
 public class StatsUtils {
 
+  private static final Log LOG = LogFactory.getLog(StatsUtils.class.getName());
+
   /**
    * Collect table, partition and column level statistics
-   *
    * @param conf
    *          - hive configuration
    * @param partList
@@ -86,26 +89,38 @@ public class StatsUtils {
     List<String> neededColumns = tableScanOperator.getNeededColumns();
     String dbName = table.getDbName();
     String tabName = table.getTableName();
-    boolean fetchColStats = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_FETCH_COLUMN_STATS);
+    boolean fetchColStats =
+        HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_FETCH_COLUMN_STATS);
+    float deserFactor =
+        HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVE_STATS_DESERIALIZATION_FACTOR);
 
     if (!table.isPartitioned()) {
       long nr = getNumRows(dbName, tabName);
-      long rds = getRawDataSize(dbName, tabName);
-      if (rds <= 0) {
-        rds = getTotalSize(dbName, tabName);
+      long ds = getRawDataSize(dbName, tabName);
+      if (ds <= 0) {
+        ds = getTotalSize(dbName, tabName);
 
         // if data size is still 0 then get file size
-        if (rds <= 0) {
-          rds = getFileSizeForTable(conf, table);
+        if (ds <= 0) {
+          ds = getFileSizeForTable(conf, table);
         }
+
+        ds = (long) (ds * deserFactor);
       }
 
       // number of rows -1 means that statistics from metastore is not reliable
+      // and 0 means statistics gathering is disabled
       if (nr <= 0) {
-        nr = 0;
+        int avgRowSize = estimateRowSizeFromSchema(conf, schema, neededColumns);
+        if (avgRowSize > 0) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Estimated average row size: " + avgRowSize);
+          }
+          nr = ds / avgRowSize;
+        }
       }
       stats.setNumRows(nr);
-      stats.setDataSize(rds);
+      stats.setDataSize(ds);
 
       List<ColStatistics> colStats = Lists.newArrayList();
       if (fetchColStats) {
@@ -126,7 +141,7 @@ public class StatsUtils {
 
       if (!checkIfColStatsAvailable(colStats)) {
         // if there is column projection and if we do not have stats then mark
-        // it as NONE. Else we will have stats for const/udf columns
+        // it as NONE. Else we will have estimated stats for const/udf columns
         if (!neededColumns.isEmpty()) {
           stats.setColumnStatsState(Statistics.State.NONE);
         } else {
@@ -144,31 +159,56 @@ public class StatsUtils {
           partNames.add(part.getName());
         }
 
-        List<Long> rowCounts = getBasicStatForPartitions(table, partNames,
-            StatsSetupConst.ROW_COUNT);
-        List<Long> dataSizes = getBasicStatForPartitions(table, partNames,
-            StatsSetupConst.RAW_DATA_SIZE);
+        List<Long> rowCounts =
+            getBasicStatForPartitions(table, partNames, StatsSetupConst.ROW_COUNT);
+        List<Long> dataSizes =
+            getBasicStatForPartitions(table, partNames, StatsSetupConst.RAW_DATA_SIZE);
 
         long nr = getSumIgnoreNegatives(rowCounts);
-        long rds = getSumIgnoreNegatives(dataSizes);
-        if (rds <= 0) {
+        long ds = getSumIgnoreNegatives(dataSizes);
+        if (ds <= 0) {
           dataSizes = getBasicStatForPartitions(table, partNames, StatsSetupConst.TOTAL_SIZE);
-          rds = getSumIgnoreNegatives(dataSizes);
+          ds = getSumIgnoreNegatives(dataSizes);
 
           // if data size still could not be determined, then fall back to filesytem to get file
           // sizes
-          if (rds <= 0) {
+          if (ds <= 0) {
             dataSizes = getFileSizeForPartitions(conf, partList.getNotDeniedPartns());
           }
-          rds = getSumIgnoreNegatives(dataSizes);
+          ds = getSumIgnoreNegatives(dataSizes);
+
+          ds = (long) (ds * deserFactor);
         }
 
-        // number of rows -1 means that statistics from metastore is not reliable
-        if (nr <= 0) {
-          nr = 0;
+        int avgRowSize = estimateRowSizeFromSchema(conf, schema, neededColumns);
+        if (avgRowSize > 0) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Estimated average row size: " + avgRowSize);
+          }
+
+          for (int i = 0; i < rowCounts.size(); i++) {
+            long rc = rowCounts.get(i);
+            long s = dataSizes.get(i);
+            if (rc <= 0 && s > 0) {
+              rc = s / avgRowSize;
+              rowCounts.set(i, rc);
+            }
+
+            if (s <= 0 && rc > 0) {
+              s = rc * avgRowSize;
+              dataSizes.set(i, s);
+            }
+          }
+          nr = getSumIgnoreNegatives(rowCounts);
+          ds = getSumIgnoreNegatives(dataSizes);
+
+          // number of rows -1 means that statistics from metastore is not reliable
+          if (nr <= 0) {
+            nr = ds / avgRowSize;
+          }
         }
         stats.addToNumRows(nr);
-        stats.addToDataSize(rds);
+        stats.addToDataSize(ds);
 
         // if atleast a partition does not contain row count then mark basic stats state as PARTIAL
         if (containsNonPositives(rowCounts)) {
@@ -187,7 +227,7 @@ public class StatsUtils {
             stats.updateColumnStatsState(Statistics.State.COMPLETE);
           } else {
             // if there is column projection and if we do not have stats then mark
-            // it as NONE. Else we will have stats for const/udf columns
+            // it as NONE. Else we will have estimated stats for const/udf columns
             if (!neededColumns.isEmpty()) {
               stats.updateColumnStatsState(Statistics.State.NONE);
             } else {
@@ -202,9 +242,40 @@ public class StatsUtils {
     return stats;
   }
 
+  public static int estimateRowSizeFromSchema(HiveConf conf, List<ColumnInfo> schema,
+      List<String> neededColumns) {
+    int avgRowSize = 0;
+    for (String neededCol : neededColumns) {
+      ColumnInfo ci = getColumnInfoForColumn(neededCol, schema);
+      ObjectInspector oi = ci.getObjectInspector();
+      String colType = ci.getTypeName();
+      if (colType.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)
+          || colType.equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME)
+          || colType.startsWith(serdeConstants.VARCHAR_TYPE_NAME)
+          || colType.startsWith(serdeConstants.CHAR_TYPE_NAME)
+          || colType.startsWith(serdeConstants.LIST_TYPE_NAME)
+          || colType.startsWith(serdeConstants.MAP_TYPE_NAME)
+          || colType.startsWith(serdeConstants.STRUCT_TYPE_NAME)
+          || colType.startsWith(serdeConstants.UNION_TYPE_NAME)) {
+        avgRowSize += getAvgColLenOfVariableLengthTypes(conf, oi, colType);
+      } else {
+        avgRowSize += getAvgColLenOfFixedLengthTypes(colType);
+      }
+    }
+    return avgRowSize;
+  }
+
+  private static ColumnInfo getColumnInfoForColumn(String neededCol, List<ColumnInfo> schema) {
+    for (ColumnInfo ci : schema) {
+      if (ci.getInternalName().equalsIgnoreCase(neededCol)) {
+        return ci;
+      }
+    }
+    return null;
+  }
+
   /**
    * Find the bytes on disk occupied by a table
-   *
    * @param conf
    *          - hive conf
    * @param table
@@ -225,7 +296,6 @@ public class StatsUtils {
 
   /**
    * Find the bytes on disks occupied by list of partitions
-   *
    * @param conf
    *          - hive conf
    * @param parts
@@ -259,7 +329,6 @@ public class StatsUtils {
 
   /**
    * Get sum of all values in the list that are >0
-   *
    * @param vals
    *          - list of values
    * @return sum
@@ -276,7 +345,6 @@ public class StatsUtils {
 
   /**
    * Get the partition level columns statistics from metastore for all the needed columns
-   *
    * @param table
    *          - table object
    * @param part
@@ -312,7 +380,6 @@ public class StatsUtils {
 
   /**
    * Get the partition level columns statistics from metastore for a specific column
-   *
    * @param dbName
    *          - database name
    * @param tabName
@@ -326,8 +393,8 @@ public class StatsUtils {
   public static ColStatistics getParitionColumnStatsForColumn(String dbName, String tabName,
       String partName, String colName) {
     try {
-      ColumnStatistics colStats = Hive.get().getPartitionColumnStatistics(dbName, tabName,
-          partName, colName);
+      ColumnStatistics colStats =
+          Hive.get().getPartitionColumnStatistics(dbName, tabName, partName, colName);
       if (colStats != null) {
         return getColStatistics(colStats.getStatsObj().get(0), tabName, colName);
       }
@@ -339,7 +406,6 @@ public class StatsUtils {
 
   /**
    * Will return true if column statistics for atleast one column is available
-   *
    * @param colStats
    *          - column stats
    * @return
@@ -355,7 +421,6 @@ public class StatsUtils {
 
   /**
    * Get table level column stats for specified column
-   *
    * @param dbName
    *          - database name
    * @param tableName
@@ -380,7 +445,6 @@ public class StatsUtils {
 
   /**
    * Convert ColumnStatisticsObj to ColStatistics
-   *
    * @param cso
    *          - ColumnStatisticsObj
    * @param tabName
@@ -446,7 +510,6 @@ public class StatsUtils {
 
   /**
    * Get table level column statistics from metastore for needed columns
-   *
    * @param table
    *          - table
    * @param schema
@@ -479,7 +542,6 @@ public class StatsUtils {
 
   /**
    * Get the raw data size of variable length data types
-   *
    * @param conf
    *          - hive conf
    * @param oi
@@ -533,10 +595,11 @@ public class StatsUtils {
         return coi.getWritableConstantValue().toString().length();
       } else if (oi instanceof WritableConstantHiveVarcharObjectInspector) {
 
-        WritableConstantHiveVarcharObjectInspector wcsoi = (WritableConstantHiveVarcharObjectInspector) oi;
+        WritableConstantHiveVarcharObjectInspector wcsoi =
+            (WritableConstantHiveVarcharObjectInspector) oi;
         return wcsoi.getWritableConstantValue().toString().length();
       } else if (oi instanceof WritableHiveVarcharObjectInspector) {
-        return ((WritableHiveVarcharObjectInspector)oi).getMaxLength();
+        return ((WritableHiveVarcharObjectInspector) oi).getMaxLength();
       }
     } else if (colType.startsWith(serdeConstants.CHAR_TYPE_NAME)) {
 
@@ -552,7 +615,8 @@ public class StatsUtils {
         return coi.getWritableConstantValue().toString().length();
       } else if (oi instanceof WritableConstantHiveCharObjectInspector) {
 
-        WritableConstantHiveCharObjectInspector wcsoi = (WritableConstantHiveCharObjectInspector) oi;
+        WritableConstantHiveCharObjectInspector wcsoi =
+            (WritableConstantHiveCharObjectInspector) oi;
         return wcsoi.getWritableConstantValue().toString().length();
       } else if (oi instanceof WritableHiveCharObjectInspector) {
         return ((WritableHiveCharObjectInspector) oi).getMaxLength();
@@ -592,7 +656,6 @@ public class StatsUtils {
 
   /**
    * Get the size of complex data types
-   *
    * @param conf
    *          - hive conf
    * @param oi
@@ -690,7 +753,6 @@ public class StatsUtils {
 
   /**
    * Get size of fixed length primitives
-   *
    * @param colType
    *          - column type
    * @return raw data size
@@ -718,7 +780,6 @@ public class StatsUtils {
 
   /**
    * Get the size of arrays of primitive types
-   *
    * @param colType
    *          - column type
    * @param length
@@ -752,7 +813,6 @@ public class StatsUtils {
 
   /**
    * Estimate the size of map object
-   *
    * @param scmoi
    *          - object inspector
    * @return size of map
@@ -774,7 +834,6 @@ public class StatsUtils {
 
   /**
    * Get size of primitive data types based on their respective writable object inspector
-   *
    * @param oi
    *          - object inspector
    * @param value
@@ -817,7 +876,6 @@ public class StatsUtils {
 
   /**
    * Get column statistics from parent statistics.
-   *
    * @param conf
    *          - hive conf
    * @param parentStats
@@ -829,8 +887,7 @@ public class StatsUtils {
    * @return column statistics
    */
   public static List<ColStatistics> getColStatisticsFromExprMap(HiveConf conf,
-      Statistics parentStats,
-      Map<String, ExprNodeDesc> colExprMap, RowSchema rowSchema) {
+      Statistics parentStats, Map<String, ExprNodeDesc> colExprMap, RowSchema rowSchema) {
 
     List<ColStatistics> cs = Lists.newArrayList();
     if (colExprMap != null) {
@@ -856,7 +913,6 @@ public class StatsUtils {
 
   /**
    * Get column statistics expression nodes
-   *
    * @param conf
    *          - hive conf
    * @param parentStats
@@ -963,7 +1019,6 @@ public class StatsUtils {
 
   /**
    * Get number of rows of a give table
-   *
    * @param dbName
    *          - database name
    * @param tabName
@@ -976,7 +1031,6 @@ public class StatsUtils {
 
   /**
    * Get raw data size of a give table
-   *
    * @param dbName
    *          - database name
    * @param tabName
@@ -989,7 +1043,6 @@ public class StatsUtils {
 
   /**
    * Get total size of a give table
-   *
    * @param dbName
    *          - database name
    * @param tabName
@@ -1002,7 +1055,6 @@ public class StatsUtils {
 
   /**
    * Get basic stats of table
-   *
    * @param dbName
    *          - database name
    * @param tabName
@@ -1035,7 +1087,6 @@ public class StatsUtils {
 
   /**
    * Get basic stats of partitions
-   *
    * @param table
    *          - table
    * @param partNames
@@ -1072,7 +1123,6 @@ public class StatsUtils {
 
   /**
    * Compute raw data size from column statistics
-   *
    * @param numRows
    *          - number of rows
    * @param colStats
@@ -1130,7 +1180,6 @@ public class StatsUtils {
 
   /**
    * Remove KEY/VALUE prefix from column name
-   *
    * @param colName
    *          - column name
    * @return column name
@@ -1146,7 +1195,6 @@ public class StatsUtils {
 
   /**
    * Returns fully qualified name of column
-   *
    * @param tabName
    * @param colName
    * @return
@@ -1157,7 +1205,6 @@ public class StatsUtils {
 
   /**
    * Returns fully qualified name of column
-   *
    * @param dbName
    * @param tabName
    * @param colName
@@ -1169,7 +1216,6 @@ public class StatsUtils {
 
   /**
    * Returns fully qualified name of column
-   *
    * @param dbName
    * @param tabName
    * @param partName
@@ -1193,7 +1239,6 @@ public class StatsUtils {
 
   /**
    * Try to get fully qualified column name from expression node
-   *
    * @param keyExprs
    *          - expression nodes
    * @param map

Modified: hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_filter.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_filter.q?rev=1548331&r1=1548330&r2=1548331&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_filter.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_filter.q Thu Dec  5 23:58:22 2013
@@ -27,7 +27,7 @@ analyze table loc_orc compute statistics
 -- numRows: 1 rawDataSize: 102
 explain extended select * from loc_orc where state='OH';
 
--- not equals comparison shouldn't affect number of rows. rawDataSize is 792 and not 796 because of rounding off issue with avgColLen. avgColLen uses integers and not double.
+-- not equals comparison shouldn't affect number of rows
 -- numRows: 8 rawDataSize: 804
 explain extended select * from loc_orc where state!='OH';
 explain extended select * from loc_orc where state<>'OH';

Modified: hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_part.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_part.q?rev=1548331&r1=1548330&r2=1548331&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_part.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/annotate_stats_part.q Thu Dec  5 23:58:22 2013
@@ -77,3 +77,9 @@ explain extended select state,locid from
 
 -- basicStatState: COMPLETE colStatState: PARTIAL
 explain extended select * from loc_orc;
+
+-- This is to test filter expression evaluation on partition column
+-- numRows: 2 dataSize: 8 basicStatState: COMPLETE colStatState: COMPLETE
+explain extended select locid from loc_orc where locid>0 and year=2001;
+explain extended select locid,year from loc_orc where locid>0 and year=2001;
+explain extended select * from (select locid,year from loc_orc) test where locid>0 and year=2001;

Modified: hive/trunk/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/annotate_stats_filter.q.out?rev=1548331&r1=1548330&r2=1548331&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/annotate_stats_filter.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/annotate_stats_filter.q.out Thu Dec  5 23:58:22 2013
@@ -121,7 +121,7 @@ STAGE PLANS:
                   expr: (state = 'OH')
                   type: boolean
               Statistics:
-                  numRows: 4 dataSize: 396 basicStatsState: COMPLETE colStatsState: NONE
+                  numRows: 4 dataSize: 398 basicStatsState: COMPLETE colStatsState: NONE
               Select Operator
                 expressions:
                       expr: state
@@ -134,14 +134,14 @@ STAGE PLANS:
                       type: int
                 outputColumnNames: _col0, _col1, _col2, _col3
                 Statistics:
-                    numRows: 4 dataSize: 396 basicStatsState: COMPLETE colStatsState: NONE
+                    numRows: 4 dataSize: 398 basicStatsState: COMPLETE colStatsState: NONE
                 File Output Operator
                   compressed: false
                   GlobalTableId: 0
 #### A masked pattern was here ####
                   NumFilesPerFileSink: 1
                   Statistics:
-                      numRows: 4 dataSize: 396 basicStatsState: COMPLETE colStatsState: NONE
+                      numRows: 4 dataSize: 398 basicStatsState: COMPLETE colStatsState: NONE
 #### A masked pattern was here ####
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -350,11 +350,11 @@ STAGE PLANS:
       limit: -1
 
 
-PREHOOK: query: -- not equals comparison shouldn't affect number of rows. rawDataSize is 792 and not 796 because of rounding off issue with avgColLen. avgColLen uses integers and not double.
+PREHOOK: query: -- not equals comparison shouldn't affect number of rows
 -- numRows: 8 rawDataSize: 804
 explain extended select * from loc_orc where state!='OH'
 PREHOOK: type: QUERY
-POSTHOOK: query: -- not equals comparison shouldn't affect number of rows. rawDataSize is 792 and not 796 because of rounding off issue with avgColLen. avgColLen uses integers and not double.
+POSTHOOK: query: -- not equals comparison shouldn't affect number of rows
 -- numRows: 8 rawDataSize: 804
 explain extended select * from loc_orc where state!='OH'
 POSTHOOK: type: QUERY
@@ -1509,7 +1509,7 @@ STAGE PLANS:
                   expr: ((year = 2001) and year is null)
                   type: boolean
               Statistics:
-                  numRows: 2 dataSize: 204 basicStatsState: COMPLETE colStatsState: COMPLETE
+                  numRows: 8 dataSize: 804 basicStatsState: COMPLETE colStatsState: COMPLETE
               Select Operator
                 expressions:
                       expr: state
@@ -1522,14 +1522,14 @@ STAGE PLANS:
                       type: int
                 outputColumnNames: _col0, _col1, _col2, _col3
                 Statistics:
-                    numRows: 2 dataSize: 204 basicStatsState: COMPLETE colStatsState: COMPLETE
+                    numRows: 8 dataSize: 804 basicStatsState: COMPLETE colStatsState: COMPLETE
                 File Output Operator
                   compressed: false
                   GlobalTableId: 0
 #### A masked pattern was here ####
                   NumFilesPerFileSink: 1
                   Statistics:
-                      numRows: 2 dataSize: 204 basicStatsState: COMPLETE colStatsState: COMPLETE
+                      numRows: 8 dataSize: 804 basicStatsState: COMPLETE colStatsState: COMPLETE
 #### A masked pattern was here ####
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -1759,7 +1759,7 @@ STAGE PLANS:
                   expr: (((year = 2001) and year is null) or (state = 'CA'))
                   type: boolean
               Statistics:
-                  numRows: 3 dataSize: 306 basicStatsState: COMPLETE colStatsState: COMPLETE
+                  numRows: 8 dataSize: 796 basicStatsState: COMPLETE colStatsState: COMPLETE
               Select Operator
                 expressions:
                       expr: state
@@ -1772,14 +1772,14 @@ STAGE PLANS:
                       type: int
                 outputColumnNames: _col0, _col1, _col2, _col3
                 Statistics:
-                    numRows: 3 dataSize: 306 basicStatsState: COMPLETE colStatsState: COMPLETE
+                    numRows: 8 dataSize: 796 basicStatsState: COMPLETE colStatsState: COMPLETE
                 File Output Operator
                   compressed: false
                   GlobalTableId: 0
 #### A masked pattern was here ####
                   NumFilesPerFileSink: 1
                   Statistics:
-                      numRows: 3 dataSize: 306 basicStatsState: COMPLETE colStatsState: COMPLETE
+                      numRows: 8 dataSize: 796 basicStatsState: COMPLETE colStatsState: COMPLETE
 #### A masked pattern was here ####
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
@@ -1885,7 +1885,7 @@ STAGE PLANS:
                   expr: (((year = 2001) or year is null) and (state = 'CA'))
                   type: boolean
               Statistics:
-                  numRows: 1 dataSize: 102 basicStatsState: COMPLETE colStatsState: COMPLETE
+                  numRows: 2 dataSize: 204 basicStatsState: COMPLETE colStatsState: COMPLETE
               Select Operator
                 expressions:
                       expr: state
@@ -1898,14 +1898,14 @@ STAGE PLANS:
                       type: int
                 outputColumnNames: _col0, _col1, _col2, _col3
                 Statistics:
-                    numRows: 1 dataSize: 102 basicStatsState: COMPLETE colStatsState: COMPLETE
+                    numRows: 2 dataSize: 204 basicStatsState: COMPLETE colStatsState: COMPLETE
                 File Output Operator
                   compressed: false
                   GlobalTableId: 0
 #### A masked pattern was here ####
                   NumFilesPerFileSink: 1
                   Statistics:
-                      numRows: 1 dataSize: 102 basicStatsState: COMPLETE colStatsState: COMPLETE
+                      numRows: 2 dataSize: 204 basicStatsState: COMPLETE colStatsState: COMPLETE
 #### A masked pattern was here ####
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat