You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by de...@apache.org on 2022/12/13 11:17:09 UTC
[hive] branch master updated: HIVE-26221: Add histogram-based column statistics (Alessandro Solimando, Ryan Johnson, reviewed by Aman Sinha, Zhihua Deng)
This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c2ff53468b0 HIVE-26221: Add histogram-based column statistics (Alessandro Solimando, Ryan Johnson, reviewed by Aman Sinha, Zhihua Deng)
c2ff53468b0 is described below
commit c2ff53468b020496b1efbeb8f5406ee8028dfdde
Author: Alessandro Solimando <al...@gmail.com>
AuthorDate: Tue Dec 13 12:16:56 2022 +0100
HIVE-26221: Add histogram-based column statistics (Alessandro Solimando, Ryan Johnson, reviewed by Aman Sinha, Zhihua Deng)
Closes #3137
---
.../apache/hadoop/hive/common/HiveStatsUtils.java | 15 +
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +
.../cache/TestCachedStoreUpdateUsingEvents.java | 65 +-
.../UDAFTemplates/VectorUDAFComputeDsKllSketch.txt | 4 +-
.../org/apache/hadoop/hive/ql/ddl/ShowUtils.java | 85 +-
.../hive/ql/ddl/table/info/desc/DescTableDesc.java | 28 +-
.../ql/ddl/table/info/desc/DescTableOperation.java | 4 +-
.../desc/formatter/TextDescTableFormatter.java | 16 +-
.../apache/hadoop/hive/ql/exec/DDLPlanUtils.java | 124 ++-
.../calcite/stats/FilterSelectivityEstimator.java | 232 +++++-
.../stats/annotation/StatsRulesProcFactory.java | 174 +++-
.../hive/ql/parse/ColumnStatsSemanticAnalyzer.java | 44 +-
.../apache/hadoop/hive/ql/plan/ColStatistics.java | 11 +
.../hadoop/hive/ql/stats/ColStatsProcessor.java | 28 +-
.../ql/stats/ColumnStatisticsObjTranslator.java | 25 +
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 7 +
.../stats/TestFilterSelectivityEstimator.java | 459 ++++++++++
.../annotation/TestStatsRulesProcFactory.java | 634 ++++++++++++++
.../test/queries/clientpositive/stats_histogram.q | 57 ++
.../llap/explain_ddl_nested_part.q.out | 86 +-
.../llap/partition_explain_ddl.q.out | 92 +-
.../clientpositive/llap/stats_histogram.q.out | 921 +++++++++++++++++++++
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 110 +++
.../src/gen/thrift/gen-cpp/hive_metastore_types.h | 65 +-
.../hive/metastore/api/DateColumnStatsData.java | 126 ++-
.../hive/metastore/api/DecimalColumnStatsData.java | 126 ++-
.../hive/metastore/api/DoubleColumnStatsData.java | 126 ++-
.../hive/metastore/api/LongColumnStatsData.java | 126 ++-
.../metastore/api/TimestampColumnStatsData.java | 126 ++-
.../gen-php/metastore/DateColumnStatsData.php | 24 +
.../gen-php/metastore/DecimalColumnStatsData.php | 24 +
.../gen-php/metastore/DoubleColumnStatsData.php | 24 +
.../gen-php/metastore/LongColumnStatsData.php | 24 +
.../gen-php/metastore/TimestampColumnStatsData.php | 24 +
.../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 70 +-
.../src/gen/thrift/gen-rb/hive_metastore_types.rb | 20 +-
.../hadoop/hive/metastore/conf/MetastoreConf.java | 2 +
.../hadoop/hive/metastore/hive_metastore.proto | 5 +
.../src/main/thrift/hive_metastore.thrift | 21 +-
standalone-metastore/metastore-server/pom.xml | 10 +
.../common/histogram}/KllHistogramEstimator.java | 15 +-
.../histogram}/KllHistogramEstimatorFactory.java | 31 +-
.../hive/common/histogram}/kll/KllUtils.java | 11 +-
.../hadoop/hive/metastore/DirectSqlUpdateStat.java | 17 +-
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 56 +-
.../apache/hadoop/hive/metastore/ObjectStore.java | 20 +-
.../hadoop/hive/metastore/StatObjectConverter.java | 69 +-
.../aggr/BinaryColumnStatsAggregator.java | 10 +-
.../aggr/BooleanColumnStatsAggregator.java | 9 +-
.../columnstats/aggr/ColumnStatsAggregator.java | 83 ++
.../aggr/DateColumnStatsAggregator.java | 104 ++-
.../aggr/DecimalColumnStatsAggregator.java | 115 ++-
.../aggr/DoubleColumnStatsAggregator.java | 112 ++-
.../aggr/LongColumnStatsAggregator.java | 112 ++-
.../aggr/StringColumnStatsAggregator.java | 12 +-
.../aggr/TimestampColumnStatsAggregator.java | 109 ++-
.../cache/DateColumnStatsDataInspector.java | 78 ++
.../cache/DecimalColumnStatsDataInspector.java | 78 ++
.../cache/DoubleColumnStatsDataInspector.java | 78 ++
.../cache/LongColumnStatsDataInspector.java | 78 ++
.../cache/TimestampColumnStatsDataInspector.java | 79 ++
.../columnstats/merge/ColumnStatsMerger.java | 23 +
.../columnstats/merge/DateColumnStatsMerger.java | 5 +
.../merge/DecimalColumnStatsMerger.java | 5 +
.../columnstats/merge/DoubleColumnStatsMerger.java | 11 +-
.../columnstats/merge/LongColumnStatsMerger.java | 5 +
.../merge/TimestampColumnStatsMerger.java | 5 +
.../model/MPartitionColumnStatistics.java | 35 +-
.../metastore/model/MTableColumnStatistics.java | 35 +-
.../src/main/resources/package.jdo | 6 +
.../src/main/sql/derby/hive-schema-4.0.0.derby.sql | 6 +-
...pgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.derby.sql | 1 -
.../derby/upgrade-4.0.0-alpha-2-to-4.0.0.derby.sql | 4 +
.../src/main/sql/mssql/hive-schema-4.0.0.mssql.sql | 6 +-
...pgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mssql.sql | 1 -
.../mssql/upgrade-4.0.0-alpha-2-to-4.0.0.mssql.sql | 4 +
.../src/main/sql/mysql/hive-schema-4.0.0.mysql.sql | 2 +
...pgrade-4.0.0-alpha-1-to-4.0.0-alpha-2.mysql.sql | 1 -
.../mysql/upgrade-4.0.0-alpha-2-to-4.0.0.mysql.sql | 4 +
.../main/sql/oracle/hive-schema-4.0.0.oracle.sql | 6 +-
.../upgrade-4.0.0-alpha-2-to-4.0.0.oracle.sql | 4 +
.../sql/postgres/hive-schema-4.0.0.postgres.sql | 6 +-
.../upgrade-4.0.0-alpha-2-to-4.0.0.postgres.sql | 4 +
.../hadoop/hive/metastore/StatisticsTestUtils.java | 157 +++-
.../hadoop/hive/metastore/TestObjectStore.java | 28 +-
.../hive/metastore/cache/TestCachedStore.java | 542 +++++-------
.../hive/metastore/client/TestPartitionStat.java | 83 +-
.../metastore/columnstats/ColStatsBuilder.java | 23 +
.../aggr/DateColumnStatsAggregatorTest.java | 81 +-
.../aggr/DecimalColumnStatsAggregatorTest.java | 76 +-
.../aggr/DoubleColumnStatsAggregatorTest.java | 30 +-
.../aggr/LongColumnStatsAggregatorTest.java | 70 +-
.../aggr/StringColumnStatsAggregatorTest.java | 4 +-
.../aggr/TimestampColumnStatsAggregatorTest.java | 81 +-
.../columnstats/merge/ColumnStatsMergerTest.java | 84 ++
.../upgrade-3.1.3000-to-4.0.0.postgres.sql | 3 +
standalone-metastore/pom.xml | 12 +
97 files changed, 5734 insertions(+), 1165 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/common/HiveStatsUtils.java b/common/src/java/org/apache/hadoop/hive/common/HiveStatsUtils.java
index 10ff548969b..be57c117fe7 100644
--- a/common/src/java/org/apache/hadoop/hive/common/HiveStatsUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/HiveStatsUtils.java
@@ -81,6 +81,21 @@ public class HiveStatsUtils {
return Lists.newArrayList(fs.globStatus(pathPattern, FileUtils.HIDDEN_FILES_PATH_FILTER));
}
+ public static boolean computeHistograms(Configuration conf) {
+ return HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_KLL_ENABLE,
+ HiveConf.ConfVars.HIVE_STATS_KLL_ENABLE.defaultBoolVal);
+ }
+
+ public static int getKParamForKllSketch(Configuration conf) {
+ int k = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_KLL_K_PARAM);
+
+ if (k < 8 || k > 65535) {
+ throw new IllegalArgumentException(HiveConf.ConfVars.HIVE_STATS_KLL_K_PARAM.varname +
+ " must be in the range [8, 65535], got " + k);
+ }
+ return k;
+ }
+
public static int getNumBitVectorsForNDVEstimation(Configuration conf) throws Exception {
int numBitVectors;
float percentageError = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVE_STATS_NDV_ERROR);
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index b3b715a81ae..e5ea252aa73 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2824,6 +2824,12 @@ public class HiveConf extends Configuration {
"higher compute cost. (NDV means the number of distinct values.). It only affects the FM-Sketch \n" +
"(not the HLL algorithm which is the default), where it computes the number of necessary\n" +
" bitvectors to achieve the accuracy."),
+ HIVE_STATS_KLL_K_PARAM("hive.stats.kll.k.param", 200,
+ "The parameter k affects the accuracy and the size of KLL sketches.\n"
+ + "Larger values of k result in smaller values of normalized rank error.\n"
+ + "The default of 200 yields approximately an error of 1.65%."),
+ HIVE_STATS_KLL_ENABLE("hive.stats.kll.enable", false,
+ "Whether to compute KLL sketches to enable histogram statistics."),
HIVE_STATS_ESTIMATORS_ENABLE("hive.stats.estimators.enable", true,
"Estimators are able to provide more accurate column statistic infos for UDF results."),
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStoreUpdateUsingEvents.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStoreUpdateUsingEvents.java
index 93b4f1dd715..dd08d8aa109 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStoreUpdateUsingEvents.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStoreUpdateUsingEvents.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.metastore.*;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.metastore.columnstats.ColStatsBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
@@ -41,10 +42,17 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.apache.hadoop.hive.metastore.StatisticsTestUtils.assertEqualStatistics;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
public class TestCachedStoreUpdateUsingEvents {
+ // the following constants are set as column statistics, and they are not expected to change,
+ // this is to keep the method signatures shorter, update is tested on a subset of the fields
+ private final static double LOW_VALUE = 50000.21;
+ private final static long NUM_NULLS = 3;
+ private final static long NUM_DVS = 22;
+ private final static long MAX_COL_LEN = 102;
private RawStore rawStore;
private SharedCache sharedCache;
private Configuration conf;
@@ -63,6 +71,8 @@ public class TestCachedStoreUpdateUsingEvents {
MetastoreConf.setBoolVar(conf, ConfVars.HIVE_TXN_STATS_ENABLED, true);
MetastoreConf.setBoolVar(conf, ConfVars.AGGREGATE_STATS_CACHE_ENABLED, false);
MetastoreConf.setBoolVar(conf, ConfVars.REPLCMENABLED, true);
+ MetastoreConf.setBoolVar(conf, ConfVars.STATS_FETCH_BITVECTOR, true);
+ MetastoreConf.setBoolVar(conf, ConfVars.STATS_FETCH_KLL, true);
MetastoreConf.setVar(conf, ConfVars.REPLCMDIR, "cmroot");
MetaStoreTestUtils.setConfForStandloneMode(conf);
@@ -555,8 +565,8 @@ public class TestCachedStoreUpdateUsingEvents {
sharedCache.getSdCache().clear();
}
- private void updateTableColStats(String dbName, String tblName, String[] colName,
- double highValue, double avgColLen, boolean isTxnTable) throws Throwable {
+ private void updateTableColStats(String dbName, String tblName, String[] colName, double highValue, double avgColLen,
+ boolean isTxnTable) throws Throwable {
long writeId = -1;
String validWriteIds = null;
if (isTxnTable) {
@@ -594,8 +604,8 @@ public class TestCachedStoreUpdateUsingEvents {
verifyStatString(colStatsCache.getStatsObj().get(0), colName[1], avgColLen);
}
- private void updatePartColStats(String dbName, String tblName, boolean isTxnTable, String[] colName,
- String partName, double highValue, double avgColLen) throws Throwable {
+ private void updatePartColStats(String dbName, String tblName, boolean isTxnTable, String[] colName, String partName,
+ double highValue, double avgColLen) throws Throwable {
long writeId = -1;
String validWriteIds = null;
List<Long> txnIds = null;
@@ -662,11 +672,7 @@ public class TestCachedStoreUpdateUsingEvents {
}
private List<ColumnStatisticsObj> getStatsObjects(String dbName, String tblName, String[] colName,
- double highValue, double avgColLen) throws Throwable {
- double lowValue = 50000.21;
- long numNulls = 3;
- long numDVs = 22;
- long maxColLen = 102;
+ double highValue, double avgColLen) {
boolean isTblLevel = true;
String partName = null;
List<ColumnStatisticsObj> statsObjs = new ArrayList<>();
@@ -681,14 +687,8 @@ public class TestCachedStoreUpdateUsingEvents {
statsObj.setColName(colName[0]);
statsObj.setColType(colType[0]);
- ColumnStatisticsData statsData = new ColumnStatisticsData();
- DoubleColumnStatsData numericStats = new DoubleColumnStatsData();
- statsData.setDoubleStats(numericStats);
-
- statsData.getDoubleStats().setHighValue(highValue);
- statsData.getDoubleStats().setLowValue(lowValue);
- statsData.getDoubleStats().setNumDVs(numDVs);
- statsData.getDoubleStats().setNumNulls(numNulls);
+ ColumnStatisticsData statsData = new ColStatsBuilder<>(double.class).numNulls(NUM_NULLS).numDVs(NUM_DVS)
+ .low(LOW_VALUE).high(highValue).hll(1.1, 2.2, 3.3).kll(1.1, 2.2, 3.3).build();
statsObj.setStatsData(statsData);
statsObjs.add(statsObj);
@@ -697,13 +697,8 @@ public class TestCachedStoreUpdateUsingEvents {
statsObj.setColName(colName[1]);
statsObj.setColType(colType[1]);
- statsData = new ColumnStatisticsData();
- StringColumnStatsData stringStats = new StringColumnStatsData();
- statsData.setStringStats(stringStats);
- statsData.getStringStats().setAvgColLen(avgColLen);
- statsData.getStringStats().setMaxColLen(maxColLen);
- statsData.getStringStats().setNumDVs(numDVs);
- statsData.getStringStats().setNumNulls(numNulls);
+ statsData = new ColStatsBuilder<>(String.class).numNulls(NUM_NULLS).numDVs(NUM_DVS)
+ .avgColLen(avgColLen).maxColLen(MAX_COL_LEN).hll("1.1", "2.2", "3.3").build();
statsObj.setStatsData(statsData);
statsObjs.add(statsObj);
@@ -711,25 +706,17 @@ public class TestCachedStoreUpdateUsingEvents {
}
private void verifyStatDouble(ColumnStatisticsObj colStats, String colName, double highValue) {
- double lowValue = 50000.21;
- long numNulls = 3;
- long numDVs = 22;
Assert.assertEquals(colStats.getColName(), colName);
- Assert.assertEquals(colStats.getStatsData().getDoubleStats().getLowValue(), lowValue, 0.01);
- Assert.assertEquals(colStats.getStatsData().getDoubleStats().getHighValue(), highValue, 0.01);
- Assert.assertEquals(colStats.getStatsData().getDoubleStats().getNumNulls(), numNulls);
- Assert.assertEquals(colStats.getStatsData().getDoubleStats().getNumDVs(), numDVs);
+ ColumnStatisticsData expectedColStats = new ColStatsBuilder<>(double.class).numNulls(NUM_NULLS).numDVs(NUM_DVS)
+ .low(LOW_VALUE).high(highValue).hll(1.1, 2.2, 3.3).kll(1.1, 2.2, 3.3).build();
+ Assert.assertEquals(expectedColStats, colStats.getStatsData());
}
private void verifyStatString(ColumnStatisticsObj colStats, String colName, double avgColLen) {
- long numNulls = 3;
- long numDVs = 22;
- long maxColLen = 102;
- Assert.assertEquals(colStats.getColName(), colName);
- Assert.assertEquals(colStats.getStatsData().getStringStats().getMaxColLen(), maxColLen);
- Assert.assertEquals(colStats.getStatsData().getStringStats().getAvgColLen(), avgColLen, 0.01);
- Assert.assertEquals(colStats.getStatsData().getStringStats().getNumNulls(), numNulls);
- Assert.assertEquals(colStats.getStatsData().getStringStats().getNumDVs(), numDVs);
+ Assert.assertEquals(colName, colStats.getColName());
+ ColumnStatisticsData expectedColStats = new ColStatsBuilder<>(String.class).numNulls(NUM_NULLS).numDVs(NUM_DVS)
+ .avgColLen(avgColLen).maxColLen(MAX_COL_LEN).hll("1.1", "2.2", "3.3").build();
+ Assert.assertEquals(expectedColStats, colStats.getStatsData());
}
private void verifyStat(List<ColumnStatisticsObj> colStats, String[] colName, double highValue, double avgColLen) {
diff --git a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFComputeDsKllSketch.txt b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFComputeDsKllSketch.txt
index 7ab07d5bd4f..076d1732019 100644
--- a/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFComputeDsKllSketch.txt
+++ b/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFComputeDsKllSketch.txt
@@ -18,8 +18,8 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates;
-import org.apache.hadoop.hive.ql.udf.datasketches.kll.KllHistogramEstimator;
-import org.apache.hadoop.hive.ql.udf.datasketches.kll.KllHistogramEstimatorFactory;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimatorFactory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/ShowUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/ShowUtils.java
index 9968d7ef3c6..0d52c852895 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/ShowUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/ShowUtils.java
@@ -21,11 +21,15 @@ package org.apache.hadoop.hive.ql.ddl;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
+import org.apache.datasketches.kll.KllFloatsSketch;
+import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
@@ -40,6 +44,7 @@ import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
import org.apache.hadoop.hive.metastore.api.TimestampColumnStatsData;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hive.common.util.HiveStringUtils;
@@ -50,6 +55,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
+import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -57,6 +63,7 @@ import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.function.Function;
/**
* Utilities for SHOW ... commands.
@@ -131,9 +138,15 @@ public final class ShowUtils {
}
}
-
+ // kept for backward compatibility since it's a public static method
+ @SuppressWarnings("unused")
public static String[] extractColumnValues(FieldSchema column, boolean isColumnStatsAvailable,
ColumnStatisticsObj columnStatisticsObj) {
+ return extractColumnValues(column, isColumnStatsAvailable, columnStatisticsObj, false);
+ }
+
+ public static String[] extractColumnValues(FieldSchema column, boolean isColumnStatsAvailable,
+ ColumnStatisticsObj columnStatisticsObj, boolean histogramEnabled) {
List<String> values = new ArrayList<>();
values.add(column.getName());
values.add(column.getType());
@@ -146,48 +159,75 @@ public final class ShowUtils {
values.addAll(Lists.newArrayList("", "", "" + binaryStats.getNumNulls(), "",
"" + binaryStats.getAvgColLen(), "" + binaryStats.getMaxColLen(), "", "",
convertToString(binaryStats.getBitVectors())));
+ if (histogramEnabled) {
+ values.add("");
+ }
} else if (statsData.isSetStringStats()) {
StringColumnStatsData stringStats = statsData.getStringStats();
values.addAll(Lists.newArrayList("", "", "" + stringStats.getNumNulls(), "" + stringStats.getNumDVs(),
"" + stringStats.getAvgColLen(), "" + stringStats.getMaxColLen(), "", "",
convertToString(stringStats.getBitVectors())));
+ if (histogramEnabled) {
+ values.add("");
+ }
} else if (statsData.isSetBooleanStats()) {
BooleanColumnStatsData booleanStats = statsData.getBooleanStats();
values.addAll(Lists.newArrayList("", "", "" + booleanStats.getNumNulls(), "", "", "",
"" + booleanStats.getNumTrues(), "" + booleanStats.getNumFalses(),
convertToString(booleanStats.getBitVectors())));
+ if (histogramEnabled) {
+ values.add("");
+ }
} else if (statsData.isSetDecimalStats()) {
DecimalColumnStatsData decimalStats = statsData.getDecimalStats();
values.addAll(Lists.newArrayList(convertToString(decimalStats.getLowValue()),
convertToString(decimalStats.getHighValue()), "" + decimalStats.getNumNulls(),
"" + decimalStats.getNumDVs(), "", "", "", "", convertToString(decimalStats.getBitVectors())));
+ if (histogramEnabled) {
+ values.add(convertHistogram(statsData.getDecimalStats().getHistogram(), statsData.getSetField()));
+ }
} else if (statsData.isSetDoubleStats()) {
DoubleColumnStatsData doubleStats = statsData.getDoubleStats();
values.addAll(Lists.newArrayList("" + doubleStats.getLowValue(), "" + doubleStats.getHighValue(),
"" + doubleStats.getNumNulls(), "" + doubleStats.getNumDVs(), "", "", "", "",
convertToString(doubleStats.getBitVectors())));
+ if (histogramEnabled) {
+ values.add(convertHistogram(statsData.getDoubleStats().getHistogram(), statsData.getSetField()));
+ }
} else if (statsData.isSetLongStats()) {
LongColumnStatsData longStats = statsData.getLongStats();
values.addAll(Lists.newArrayList("" + longStats.getLowValue(), "" + longStats.getHighValue(),
"" + longStats.getNumNulls(), "" + longStats.getNumDVs(), "", "", "", "",
convertToString(longStats.getBitVectors())));
+ if (histogramEnabled) {
+ values.add(convertHistogram(statsData.getLongStats().getHistogram(), statsData.getSetField()));
+ }
} else if (statsData.isSetDateStats()) {
DateColumnStatsData dateStats = statsData.getDateStats();
values.addAll(Lists.newArrayList(convertToString(dateStats.getLowValue()),
convertToString(dateStats.getHighValue()), "" + dateStats.getNumNulls(), "" + dateStats.getNumDVs(),
"", "", "", "", convertToString(dateStats.getBitVectors())));
+ if (histogramEnabled) {
+ values.add(convertHistogram(statsData.getDateStats().getHistogram(), statsData.getSetField()));
+ }
} else if (statsData.isSetTimestampStats()) {
TimestampColumnStatsData timestampStats = statsData.getTimestampStats();
values.addAll(Lists.newArrayList(convertToString(timestampStats.getLowValue()),
convertToString(timestampStats.getHighValue()), "" + timestampStats.getNumNulls(),
"" + timestampStats.getNumDVs(), "", "", "", "", convertToString(timestampStats.getBitVectors())));
+ if (histogramEnabled) {
+ values.add(convertHistogram(statsData.getTimestampStats().getHistogram(), statsData.getSetField()));
+ }
}
} else {
values.addAll(Lists.newArrayList("", "", "", "", "", "", "", "", ""));
+ if (histogramEnabled) {
+ values.add("");
+ }
}
}
- values.add(column.getComment() != null ? column.getComment() : "");
+ values.add(column.getComment() == null ? "" : column.getComment());
return values.toArray(new String[0]);
}
@@ -209,6 +249,47 @@ public final class ShowUtils {
return writableValue.toString();
}
+ // converts the histogram from its serialization to a string representing its quantiles
+ private static String convertHistogram(byte[] buffer, ColumnStatisticsData._Fields field) {
+ if (buffer == null || buffer.length == 0) {
+ return "";
+ }
+ final KllFloatsSketch kll = KllFloatsSketch.heapify(Memory.wrap(buffer));
+ // to keep the visualization compact, we print only the quartiles (Q1, Q2 and Q3),
+ // as min and max are displayed as separate statistics already
+ final float[] quantiles = kll.getQuantiles(new double[]{ 0.25, 0.5, 0.75 });
+
+ Function<Float, Object> converter;
+
+ switch(field) {
+ case DATE_STATS:
+ converter = f -> Date.valueOf(Timestamp.ofEpochSecond(f.longValue(), 0, getZoneIdFromConf()).toString());
+ break;
+ case DECIMAL_STATS:
+ converter = HiveDecimal::create;
+ break;
+ case DOUBLE_STATS:
+ converter = f -> f;
+ break;
+ case TIMESTAMP_STATS:
+ converter = f -> Timestamp.ofEpochSecond(f.longValue(), 0, getZoneIdFromConf());
+ break;
+ case LONG_STATS:
+ converter = Float::longValue;
+ break;
+ default:
+ return "";
+ }
+
+ return kll.isEmpty() ? "" : "Q1: " + converter.apply(quantiles[0]) + ", Q2: "
+ + converter.apply(quantiles[1]) + ", Q3: " + converter.apply(quantiles[2]);
+ }
+
+ private static ZoneId getZoneIdFromConf() {
+ return SessionState.get() == null ? new HiveConf().getLocalTimeZone()
+ : SessionState.get().getConf().getLocalTimeZone();
+ }
+
private static String convertToString(byte[] buffer) {
if (buffer == null || buffer.length == 0) {
return "";
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java
index 7c709b048d0..0354a047a90 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java
@@ -40,10 +40,18 @@ public class DescTableDesc implements DDLDesc, Serializable {
public static final String SCHEMA = "col_name,data_type,comment#string:string:string";
public static final String COLUMN_STATISTICS_SCHEMA = "column_property,value#string:string";
public static final String PARTITION_TRANSFORM_SPEC_SCHEMA = "col_name,transform_type#string:string";
- public static final List<String> COLUMN_STATISTICS_HEADERS = ImmutableList.of(
- "col_name", "data_type", "min", "max", "num_nulls", "distinct_count", "avg_col_len", "max_col_len", "num_trues",
- "num_falses", "bit_vector", "comment"
+ private static final List<String> PURE_STATISTICS_HEADERS = ImmutableList.of(
+ "min", "max", "num_nulls", "distinct_count", "avg_col_len", "max_col_len", "num_trues",
+ "num_falses", "bit_vector"
);
+ // keeping this around for backward compatibility, use "getColumnStatisticsHeaders" instead
+ @SuppressWarnings("unused")
+ public static final List<String> COLUMN_STATISTICS_HEADERS = ImmutableList.<String>builder()
+ .add("col_name")
+ .add("data_type")
+ .addAll(PURE_STATISTICS_HEADERS)
+ .add("comment")
+ .build();
private final String resFile;
private final TableName tableName;
@@ -96,4 +104,18 @@ public class DescTableDesc implements DDLDesc, Serializable {
public boolean isFormatted() {
return isFormatted;
}
+
+ public static List<String> getColumnStatisticsHeaders(boolean histogramEnabled) {
+ ImmutableList.Builder<String> builder = ImmutableList.<String>builder()
+ .add("col_name")
+ .add("data_type")
+ .addAll(PURE_STATISTICS_HEADERS);
+
+ if (histogramEnabled) {
+ builder.add("histogram");
+ }
+
+ builder.add("comment");
+ return builder.build();
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
index 6aee8dedb28..5e4f9b23425 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
@@ -244,8 +244,8 @@ public class DescTableOperation extends DDLOperation<DescTableDesc> {
StatObjectConverter.fillColumnStatisticsData(partCol.getType(), data, r == null ? null : r.minValue,
r == null ? null : r.maxValue, r == null ? null : r.minValue, r == null ? null : r.maxValue,
r == null ? null : r.minValue.toString(), r == null ? null : r.maxValue.toString(),
- cs.getNumNulls(), cs.getCountDistint(), null, cs.getAvgColLen(), cs.getAvgColLen(),
- cs.getNumTrues(), cs.getNumFalses());
+ cs.getNumNulls(), cs.getCountDistint(), null, null, cs.getAvgColLen(),
+ cs.getAvgColLen(), cs.getNumTrues(), cs.getNumFalses());
ColumnStatisticsObj cso = new ColumnStatisticsObj(partCol.getName(), partCol.getType(), data);
colStats.add(cso);
StatsSetupConst.setColumnStatsState(tableProps, colNames);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
index 5583337ce20..98fabb6c30f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.SourceTable;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.ddl.ShowUtils;
import org.apache.hadoop.hive.ql.ddl.ShowUtils.TextMetaDataTable;
import org.apache.hadoop.hive.ql.ddl.table.info.desc.DescTableDesc;
@@ -83,7 +84,7 @@ class TextDescTableFormatter extends DescTableFormatter {
Partition partition, List<FieldSchema> columns, boolean isFormatted, boolean isExtended, boolean isOutputPadded,
List<ColumnStatisticsObj> columnStats) throws HiveException {
try {
- addStatsData(out, columnPath, columns, isFormatted, columnStats, isOutputPadded);
+ addStatsData(out, conf, columnPath, columns, isFormatted, columnStats, isOutputPadded);
addPartitionData(out, conf, columnPath, table, isFormatted, isOutputPadded);
boolean isIcebergMetaTable = table.getMetaTable() != null;
@@ -131,21 +132,22 @@ class TextDescTableFormatter extends DescTableFormatter {
out.write(partitionTransformOutput.getBytes(StandardCharsets.UTF_8));
}
- private void addStatsData(DataOutputStream out, String columnPath, List<FieldSchema> columns, boolean isFormatted,
- List<ColumnStatisticsObj> columnStats, boolean isOutputPadded) throws IOException {
+ private void addStatsData(DataOutputStream out, HiveConf conf, String columnPath, List<FieldSchema> columns,
+ boolean isFormatted, List<ColumnStatisticsObj> columnStats, boolean isOutputPadded) throws IOException {
String statsData = "";
TextMetaDataTable metaDataTable = new TextMetaDataTable();
boolean needColStats = isFormatted && columnPath != null;
+ boolean histogramEnabled = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
if (needColStats) {
- metaDataTable.addRow(DescTableDesc.COLUMN_STATISTICS_HEADERS.toArray(new String[0]));
+ metaDataTable.addRow(DescTableDesc.getColumnStatisticsHeaders(histogramEnabled).toArray(new String[0]));
} else if (isFormatted && !SessionState.get().isHiveServerQuery()) {
statsData += "# ";
metaDataTable.addRow(DescTableDesc.SCHEMA.split("#")[0].split(","));
}
for (FieldSchema column : columns) {
metaDataTable.addRow(ShowUtils.extractColumnValues(column, needColStats,
- getColumnStatisticsObject(column.getName(), column.getType(), columnStats)));
+ getColumnStatisticsObject(column.getName(), column.getType(), columnStats), histogramEnabled));
}
if (needColStats) {
metaDataTable.transpose();
@@ -175,10 +177,12 @@ class TextDescTableFormatter extends DescTableFormatter {
if (CollectionUtils.isNotEmpty(partitionColumns) &&
conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) {
TextMetaDataTable metaDataTable = new TextMetaDataTable();
+ boolean histogramEnabled = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL);
partitionData += LINE_DELIM + "# Partition Information" + LINE_DELIM + "# ";
metaDataTable.addRow(DescTableDesc.SCHEMA.split("#")[0].split(","));
for (FieldSchema partitionColumn : partitionColumns) {
- metaDataTable.addRow(ShowUtils.extractColumnValues(partitionColumn, false, null));
+ metaDataTable.addRow(ShowUtils.extractColumnValues(
+ partitionColumn, false, null, histogramEnabled));
}
partitionData += metaDataTable.renderTable(isOutputPadded);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
index 649455163fe..d4d9df2f88e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
@@ -213,14 +213,23 @@ public class DDLPlanUtils {
"> <" + COL_TYPE + "> CONSTRAINT <" + CONSTRAINT_NAME + "> DEFAULT <" + DEFAULT_VALUE + "> <"
+ ENABLE + "> <" + VALIDATE + "> <" + RELY + ">;";
- private final String EXIST_BIT_VECTORS = "-- BIT VECTORS PRESENT FOR <" + DATABASE_NAME + ">.<" + TABLE_NAME + "> " +
- "FOR COLUMN <" + COLUMN_NAME + "> BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS <" +
- BASE_64_VALUE + "> ";
+ private final String EXIST_BIT_VECTORS = "-- BIT VECTORS PRESENT FOR <" + DATABASE_NAME + ">.<" + TABLE_NAME + "> "
+ + "FOR COLUMN <" + COLUMN_NAME + "> BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS <"
+ + BASE_64_VALUE + "> ";
- private final String EXIST_BIT_VECTORS_PARTITIONED = "-- BIT VECTORS PRESENT FOR <" + DATABASE_NAME + ">.<" +
- TABLE_NAME + "> PARTITION <" + PARTITION_NAME + "> FOR COLUMN <"
- + COLUMN_NAME + "> BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS <" +
- BASE_64_VALUE + "> ";
+ private final String EXIST_BIT_VECTORS_PARTITIONED = "-- BIT VECTORS PRESENT FOR <" + DATABASE_NAME + ">.<"
+ + TABLE_NAME + "> PARTITION <" + PARTITION_NAME + "> FOR COLUMN <"
+ + COLUMN_NAME + "> BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS <"
+ + BASE_64_VALUE + "> ";
+
+ private final String EXIST_HISTOGRAM = "-- HISTOGRAM PRESENT FOR <" + DATABASE_NAME + ">.<" + TABLE_NAME + "> "
+ + "FOR COLUMN <" + COLUMN_NAME + "> BUT IT IS NOT SUPPORTED YET. THE BASE64 VALUE FOR THE HISTOGRAM IS <"
+ + BASE_64_VALUE + "> ";
+
+ private final String EXIST_HISTOGRAM_PARTITIONED = "-- HISTOGRAM PRESENT FOR <" + DATABASE_NAME + ">.<"
+ + TABLE_NAME + "> PARTITION <" + PARTITION_NAME + "> FOR COLUMN <"
+ + COLUMN_NAME + "> BUT IT IS NOT SUPPORTED YET. THE BASE64 VALUE FOR THE HISTOGRAM IS <"
+ + BASE_64_VALUE + "> ";
/**
* Returns the create database query for a give database name.
@@ -395,29 +404,59 @@ public class DDLPlanUtils {
ls.add(lowValue + dc.getLowValue() + "'");
}
+ /**
+ * Checks whether the column statistics data stores histogram statistics, and returns its base64 encoding if so.
+ * @param cd the column statistics data
+ * @return the base64 encoding of the histogram statistics if existing, null otherwise.
+ */
+ public String checkHistogram(ColumnStatisticsData cd) {
+ byte[] buffer = null;
+
+ if (cd.isSetDateStats() && cd.getDateStats().isSetHistogram()) {
+ buffer = cd.getDateStats().getHistogram();
+ } else if (cd.isSetDoubleStats() && cd.getDoubleStats().isSetHistogram()) {
+ buffer = cd.getDoubleStats().getHistogram();
+ } else if (cd.isSetDecimalStats() && cd.getDecimalStats().isSetHistogram()) {
+ buffer = cd.getDecimalStats().getHistogram();
+ } else if (cd.isSetLongStats() && cd.getLongStats().isSetHistogram()) {
+ buffer = cd.getLongStats().getHistogram();
+ } else if (cd.isSetTimestampStats() && cd.getTimestampStats().isSetHistogram()) {
+ buffer = cd.getTimestampStats().getHistogram();
+ }
+
+ return encodeBytes(buffer);
+ }
+
public String checkBitVectors(ColumnStatisticsData cd) {
+ byte[] buffer = null;
+
if (cd.isSetDoubleStats() && cd.getDoubleStats().isSetBitVectors()) {
- return Base64.getEncoder().encodeToString(cd.getDoubleStats().getBitVectors());
+ buffer = cd.getDoubleStats().getBitVectors();
}
if (cd.isSetBinaryStats() && cd.getBinaryStats().isSetBitVectors()) {
- return Base64.getEncoder().encodeToString(cd.getBinaryStats().getBitVectors());
+ buffer = cd.getBinaryStats().getBitVectors();
}
if (cd.isSetStringStats() && cd.getStringStats().isSetBitVectors()) {
- return Base64.getEncoder().encodeToString(cd.getStringStats().getBitVectors());
+ buffer = cd.getStringStats().getBitVectors();
}
if (cd.isSetDateStats() && cd.getDateStats().isSetBitVectors()) {
- return Base64.getEncoder().encodeToString(cd.getDateStats().getBitVectors());
+ buffer = cd.getDateStats().getBitVectors();
}
if (cd.isSetLongStats() && cd.getLongStats().isSetBitVectors()) {
- return Base64.getEncoder().encodeToString(cd.getLongStats().getBitVectors());
+ buffer = cd.getLongStats().getBitVectors();
}
if (cd.isSetDecimalStats() && cd.getDecimalStats().isSetBitVectors()) {
- return Base64.getEncoder().encodeToString(cd.getDecimalStats().getBitVectors());
+ buffer = cd.getDecimalStats().getBitVectors();
}
- if (cd.isSetDoubleStats() && cd.getBooleanStats().isSetBitVectors()) {
- return Base64.getEncoder().encodeToString(cd.getBooleanStats().getBitVectors());
+ if (cd.isSetBooleanStats() && cd.getBooleanStats().isSetBitVectors()) {
+ buffer = cd.getBooleanStats().getBitVectors();
}
- return null;
+
+ return encodeBytes(buffer);
+ }
+
+ private String encodeBytes(byte[] buffer) {
+ return buffer == null ? null : Base64.getEncoder().encodeToString(buffer);
}
public String addAllColStats(ColumnStatisticsData columnStatisticsData) {
@@ -458,24 +497,32 @@ public class DDLPlanUtils {
*/
public List<String> getAlterTableStmtTableStatsColsAll(Table tbl)
throws HiveException {
- List<String> alterTblStmt = new ArrayList<String>();
+ List<String> alterTblStmt = new ArrayList<>();
List<String> accessedColumns = getTableColumnNames(tbl);
List<ColumnStatisticsObj> tableColumnStatistics = Hive.get().getTableColumnStatistics(tbl.getDbName(),
tbl.getTableName(),
accessedColumns,
true);
ColumnStatisticsObj[] columnStatisticsObj = tableColumnStatistics.toArray(new ColumnStatisticsObj[0]);
- for (int i = 0; i < columnStatisticsObj.length; i++) {
- alterTblStmt.add(getAlterTableStmtCol(columnStatisticsObj[i].getStatsData(),
- columnStatisticsObj[i].getColName(),
- tbl.getTableName(), tbl.getDbName()));
- String base64 = checkBitVectors(columnStatisticsObj[i].getStatsData());
- if (base64 != null) {
+ for (ColumnStatisticsObj statisticsObj : columnStatisticsObj) {
+ alterTblStmt.add(getAlterTableStmtCol(
+ statisticsObj.getStatsData(), statisticsObj.getColName(), tbl.getTableName(), tbl.getDbName()));
+ String base64BitVector = checkBitVectors(statisticsObj.getStatsData());
+ if (base64BitVector != null) {
ST command = new ST(EXIST_BIT_VECTORS);
command.add(DATABASE_NAME, tbl.getDbName());
command.add(TABLE_NAME, tbl.getTableName());
- command.add(COLUMN_NAME, columnStatisticsObj[i].getColName());
- command.add(BASE_64_VALUE, base64);
+ command.add(COLUMN_NAME, statisticsObj.getColName());
+ command.add(BASE_64_VALUE, base64BitVector);
+ alterTblStmt.add(command.render());
+ }
+ String base64Histogram = checkHistogram(statisticsObj.getStatsData());
+ if (base64Histogram != null) {
+ ST command = new ST(EXIST_HISTOGRAM);
+ command.add(DATABASE_NAME, tbl.getDbName());
+ command.add(TABLE_NAME, tbl.getTableName());
+ command.add(COLUMN_NAME, statisticsObj.getColName());
+ command.add(BASE_64_VALUE, base64Histogram);
alterTblStmt.add(command.render());
}
}
@@ -521,20 +568,27 @@ public class DDLPlanUtils {
String dbName) {
List<String> alterTableStmt = new ArrayList<String>();
ColumnStatisticsObj[] columnStatisticsObj = columnStatisticsObjList.toArray(new ColumnStatisticsObj[0]);
- for (int i = 0; i < columnStatisticsObj.length; i++) {
- alterTableStmt.add(getAlterTableStmtPartitionColStat(columnStatisticsObj[i].getStatsData(),
- columnStatisticsObj[i].getColName(),
- tblName,
- ptName,
- dbName));
- String base64 = checkBitVectors(columnStatisticsObj[i].getStatsData());
- if (base64 != null) {
+ for (ColumnStatisticsObj statisticsObj : columnStatisticsObj) {
+ alterTableStmt.add(getAlterTableStmtPartitionColStat(
+ statisticsObj.getStatsData(), statisticsObj.getColName(), tblName, ptName, dbName));
+ String base64BitVectors = checkBitVectors(statisticsObj.getStatsData());
+ if (base64BitVectors != null) {
ST command = new ST(EXIST_BIT_VECTORS_PARTITIONED);
command.add(DATABASE_NAME, dbName);
command.add(TABLE_NAME, tblName);
command.add(PARTITION_NAME, ptName);
- command.add(COLUMN_NAME, columnStatisticsObj[i].getColName());
- command.add(BASE_64_VALUE, base64);
+ command.add(COLUMN_NAME, statisticsObj.getColName());
+ command.add(BASE_64_VALUE, base64BitVectors);
+ alterTableStmt.add(command.render());
+ }
+ String base64Histogram = checkHistogram(statisticsObj.getStatsData());
+ if (base64Histogram != null) {
+ ST command = new ST(EXIST_HISTOGRAM_PARTITIONED);
+ command.add(DATABASE_NAME, dbName);
+ command.add(TABLE_NAME, tblName);
+ command.add(PARTITION_NAME, ptName);
+ command.add(COLUMN_NAME, statisticsObj.getColName());
+ command.add(BASE_64_VALUE, base64Histogram);
alterTableStmt.add(command.render());
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
index edfcb76c540..87e70e027c3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
@@ -17,7 +17,10 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.stats;
+import java.math.BigDecimal;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.GregorianCalendar;
import java.util.List;
import java.util.Set;
@@ -38,6 +41,8 @@ import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.datasketches.kll.KllFloatsSketch;
+import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfPlannerContext;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
@@ -47,6 +52,8 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.calcite.sql.fun.SqlStdOperatorTable.NOT_BETWEEN;
+
public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
protected static final Logger LOG = LoggerFactory.getLogger(FilterSelectivityEstimator.class);
@@ -90,7 +97,7 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
return 1.0;
}
- Double selectivity = null;
+ Double selectivity;
SqlKind op = getOp(call);
switch (op) {
@@ -139,10 +146,14 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
case GREATER_THAN_OR_EQUAL:
case LESS_THAN:
case GREATER_THAN: {
- selectivity = ((double) 1 / (double) 3);
+ selectivity = computeRangePredicateSelectivity(call, op);
break;
}
+ case BETWEEN:
+ selectivity = computeBetweenPredicateSelectivity(call);
+ break;
+
case IN: {
// TODO: 1) check for duplicates 2) We assume in clause values to be
// present in NDV which may not be correct (Range check can find it) 3) We
@@ -167,6 +178,122 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
return selectivity;
}
+ private double computeRangePredicateSelectivity(RexCall call, SqlKind op) {
+ final boolean isLiteralLeft = call.getOperands().get(0).getKind().equals(SqlKind.LITERAL);
+ final boolean isLiteralRight = call.getOperands().get(1).getKind().equals(SqlKind.LITERAL);
+ final boolean isInputRefLeft = call.getOperands().get(0).getKind().equals(SqlKind.INPUT_REF);
+ final boolean isInputRefRight = call.getOperands().get(1).getKind().equals(SqlKind.INPUT_REF);
+
+ if (childRel instanceof HiveTableScan && isLiteralLeft != isLiteralRight && isInputRefLeft != isInputRefRight) {
+ final HiveTableScan t = (HiveTableScan) childRel;
+ final int inputRefIndex = ((RexInputRef) call.getOperands().get(isInputRefLeft ? 0 : 1)).getIndex();
+ final List<ColStatistics> colStats = t.getColStat(Collections.singletonList(inputRefIndex));
+
+ if (!colStats.isEmpty() && isHistogramAvailable(colStats.get(0))) {
+ final KllFloatsSketch kll = KllFloatsSketch.heapify(Memory.wrap(colStats.get(0).getHistogram()));
+ final Object boundValueObject = ((RexLiteral) call.getOperands().get(isLiteralLeft ? 0 : 1)).getValue();
+ final SqlTypeName typeName = call.getOperands().get(isInputRefLeft ? 0 : 1).getType().getSqlTypeName();
+ float value = extractLiteral(typeName, boundValueObject);
+ boolean closedBound = op.equals(SqlKind.LESS_THAN_OR_EQUAL) || op.equals(SqlKind.GREATER_THAN_OR_EQUAL);
+
+ double selectivity;
+ if (op.equals(SqlKind.LESS_THAN_OR_EQUAL) || op.equals(SqlKind.LESS_THAN)) {
+ selectivity = closedBound ? lessThanOrEqualSelectivity(kll, value) : lessThanSelectivity(kll, value);
+ } else {
+ selectivity = closedBound ? greaterThanOrEqualSelectivity(kll, value) : greaterThanSelectivity(kll, value);
+ }
+
+ // selectivity does not account for null values, we multiply for the number of non-null values (getN)
+ // and we divide by the total (non-null + null values) to get the overall selectivity.
+ //
+ // Example: consider a filter "col < 3", and the following table rows:
+ // _____
+ // | col |
+ // |_____|
+ // |1 |
+ // |null |
+ // |null |
+ // |3 |
+ // |4 |
+ // -------
+ // kll.getN() would be 3, selectivity 1/3, t.getTable().getRowCount() 5
+ // so the final result would be 3 * 1/3 / 5 = 1/5, as expected.
+ return kll.getN() * selectivity / t.getTable().getRowCount();
+ }
+ }
+ return ((double) 1 / (double) 3);
+ }
+
+ private Double computeBetweenPredicateSelectivity(RexCall call) {
+ final boolean hasInputRef = call.getOperands().get(0).getKind().equals(SqlKind.INPUT_REF);
+ final boolean hasLiteralLeft = call.getOperands().get(1).getKind().equals(SqlKind.LITERAL);
+ final boolean hasLiteralRight = call.getOperands().get(2).getKind().equals(SqlKind.LITERAL);
+
+ if (childRel instanceof HiveTableScan && hasInputRef && hasLiteralLeft && hasLiteralRight) {
+ final HiveTableScan t = (HiveTableScan) childRel;
+ final int inputRefIndex = ((RexInputRef) call.getOperands().get(0)).getIndex();
+ final List<ColStatistics> colStats = t.getColStat(Collections.singletonList(inputRefIndex));
+
+ if (!colStats.isEmpty() && isHistogramAvailable(colStats.get(0))) {
+ final KllFloatsSketch kll = KllFloatsSketch.heapify(Memory.wrap(colStats.get(0).getHistogram()));
+ final SqlTypeName typeName = call.getOperands().get(0).getType().getSqlTypeName();
+ final Object leftBoundValueObject = ((RexLiteral) call.getOperands().get(1)).getValue();
+ float leftValue = extractLiteral(typeName, leftBoundValueObject);
+ final Object rightBoundValueObject = ((RexLiteral) call.getOperands().get(2)).getValue();
+ float rightValue = extractLiteral(typeName, rightBoundValueObject);
+
+ if (NOT_BETWEEN.equals(call.getOperator())) {
+ if (rightValue <= leftValue) {
+ return 1.0;
+ }
+ return lessThanSelectivity(kll, leftValue) + greaterThanSelectivity(kll, rightValue);
+ }
+ // when they are equal it's an equality predicate, we cannot handle it as "between"
+ if (Double.compare(leftValue, rightValue) != 0) {
+ return betweenSelectivity(kll, leftValue, rightValue);
+ }
+ }
+ }
+ return computeFunctionSelectivity(call);
+ }
+
+ private float extractLiteral(SqlTypeName typeName, Object boundValueObject) {
+ final String boundValueString = boundValueObject.toString();
+
+ float value;
+ switch (typeName) {
+ case TINYINT:
+ value = Byte.parseByte(boundValueString);
+ break;
+ case SMALLINT:
+ value = Short.parseShort(boundValueString);
+ break;
+ case INTEGER:
+ value = Integer.parseInt(boundValueString);
+ break;
+ case BIGINT:
+ value = Long.parseLong(boundValueString);
+ break;
+ case FLOAT:
+ value = Float.parseFloat(boundValueString);
+ break;
+ case DOUBLE:
+ value = (float) Double.parseDouble(boundValueString);
+ break;
+ case DECIMAL:
+ value = new BigDecimal(boundValueString).floatValue();
+ break;
+ case DATE:
+ case TIMESTAMP:
+ value = ((GregorianCalendar) boundValueObject).toInstant().getEpochSecond();
+ break;
+ default:
+ throw new IllegalStateException(
+ "Unsupported type for comparator selectivity evaluation using histogram: " + typeName);
+ }
+ return value;
+ }
+
/**
* NDV of "f1(x, y, z) != f2(p, q, r)" ->
* "(maxNDV(x,y,z,p,q,r) - 1)/maxNDV(x,y,z,p,q,r)".
@@ -192,9 +319,7 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
/**
* Selectivity of f(X,y,z) -> 1/maxNDV(x,y,z).
* <p>
- * Note that >, >=, <, <=, = ... are considered generic functions and uses
- * this method to find their selectivity.
- *
+ * Note that = is considered a generic function and uses this method to find its selectivity.
* @param call
* @return
*/
@@ -367,4 +492,101 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
}
return null;
}
+
+ private static double rangedSelectivity(KllFloatsSketch kll, float val1, float val2) {
+ float[] splitPoints = new float[] { val1, val2 };
+ double[] boundaries = kll.getCDF(splitPoints);
+ return boundaries[1] - boundaries[0];
+ }
+
+ /**
+ * Returns the selectivity of a predicate "column > value" in the range [0, 1]
+ * @param kll the KLL sketch for the involved column
+ * @param value the literal used in the comparison
+ * @return the selectivity of a predicate "column > value" in the range [0, 1]
+ */
+ public static double greaterThanSelectivity(KllFloatsSketch kll, float value) {
+ float max = kll.getMaxValue();
+ if (value > max) {
+ return 0;
+ }
+ float nextValue = Math.nextUp(value);
+ if (Double.compare(value, max) == 0 || Double.compare(nextValue, max) == 0) {
+ return 0;
+ }
+ return rangedSelectivity(kll, nextValue, Math.nextUp(max));
+ }
+
+ /**
+ * Returns the selectivity of a predicate "column >= value" in the range [0, 1]
+ * @param kll the KLL sketch for the involved column
+ * @param value the literal used in the comparison
+ * @return the selectivity of a predicate "column >= value" in the range [0, 1]
+ */
+ public static double greaterThanOrEqualSelectivity(KllFloatsSketch kll, float value) {
+ if (value > kll.getMaxValue()) {
+ return 0;
+ }
+ return rangedSelectivity(kll, value, Math.nextUp(kll.getMaxValue()));
+ }
+
+ /**
+ * Returns the selectivity of a predicate "column <= value" in the range [0, 1]
+ * @param kll the KLL sketch for the involved column
+ * @param value the literal used in the comparison
+ * @return the selectivity of a predicate "column <= value" in the range [0, 1]
+ */
+ public static double lessThanOrEqualSelectivity(KllFloatsSketch kll, float value) {
+ if (value < kll.getMinValue()) {
+ return 0;
+ }
+ return kll.getCDF(new float[] { Math.nextUp(value) })[0];
+ }
+
+ /**
+ * Returns the selectivity of a predicate "column < value" in the range [0, 1]
+ * @param kll the KLL sketch for the involved column
+ * @param value the literal used in the comparison
+ * @return the selectivity of a predicate "column < value" in the range [0, 1]
+ */
+ public static double lessThanSelectivity(KllFloatsSketch kll, float value) {
+ float min = kll.getMinValue();
+ if (value < min) {
+ return 0;
+ }
+ if (Double.compare(value, min) == 0 || Double.compare(Math.nextUp(value), min) == 0) {
+ return 0;
+ }
+ return kll.getCDF(new float[] { value })[0];
+ }
+
+ /**
+ * Returns the selectivity of a predicate "column BETWEEN leftValue AND rightValue" in the range [0, 1]
+ * @param kll the KLL sketch for the involved column
+ * @param leftValue the LHS literal in the BETWEEN statement
+ * @param rightValue the RHS literal in the BETWEEN statement
+ * @return the selectivity of a predicate "column BETWEEN leftValue AND rightValue" in the range [0, 1]
+ * @throws IllegalArgumentException if leftValue is equal to rightValue
+ */
+ public static double betweenSelectivity(KllFloatsSketch kll, float leftValue, float rightValue) {
+ // column >= leftValue AND column <= rightValue
+ if (rightValue < leftValue) {
+ return 0;
+ }
+ if (Double.compare(leftValue, rightValue) == 0) {
+ throw new IllegalArgumentException(
+ "Selectivity for BETWEEN leftValue AND rightValue when the two values coincide is not supported, found: "
+ + "leftValue = " + leftValue + " and rightValue = " + rightValue);
+ }
+ return rangedSelectivity(kll, Math.nextDown(leftValue), Math.nextUp(rightValue));
+ }
+
+ /**
+ * Returns true if histogram statistics are available in the column statistics information, false otherwise.
+ * @param colStats the input column statistics information
+ * @return true if histogram statistics are available in the column statistics information, false otherwise.
+ */
+ public static boolean isHistogramAvailable(ColStatistics colStats) {
+ return colStats != null && colStats.getHistogram() != null && colStats.getHistogram().length > 0;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index fd56189f2be..66f0891fecf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -34,13 +34,17 @@ import java.util.Optional;
import java.util.Set;
import java.util.Stack;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.calcite.rel.metadata.RelMdUtil;
+import org.apache.datasketches.SketchesArgumentException;
+import org.apache.datasketches.kll.KllFloatsSketch;
+import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
+import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.Timestamp;
-import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
@@ -67,6 +71,7 @@ import org.apache.hadoop.hive.ql.lib.SemanticNodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator;
import org.apache.hadoop.hive.ql.optimizer.signature.OpTreeSignature;
import org.apache.hadoop.hive.ql.parse.ColumnStatsList;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
@@ -134,6 +139,11 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.greaterThanOrEqualSelectivity;
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.greaterThanSelectivity;
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.lessThanOrEqualSelectivity;
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.lessThanSelectivity;
+
public class StatsRulesProcFactory {
private static final Logger LOG = LoggerFactory.getLogger(StatsRulesProcFactory.class.getName());
@@ -834,6 +844,38 @@ public class StatsRulesProcFactory {
return currNumRows;
}
+ final String leftValueString = leftExpression instanceof ExprNodeConstantDesc
+ ? ((ExprNodeConstantDesc) leftExpression).getValue().toString() : leftExpression.getExprString();
+ final String rightValueString = rightExpression instanceof ExprNodeConstantDesc
+ ? ((ExprNodeConstantDesc) rightExpression).getValue().toString() : rightExpression.getExprString();
+
+ try {
+ if (comparisonExpression instanceof ExprNodeColumnDesc) {
+ final ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) comparisonExpression;
+ ColStatistics cs = stats.getColumnStatisticsFromColName(columnDesc.getColumn());
+ if (FilterSelectivityEstimator.isHistogramAvailable(cs)) {
+ final KllFloatsSketch kll = KllFloatsSketch.heapify(Memory.wrap(cs.getHistogram()));
+ final String colTypeLowerCase = columnDesc.getTypeString().toLowerCase();
+ final float leftValue = extractFloatFromLiteralValue(colTypeLowerCase, leftValueString);
+ final float rightValue = extractFloatFromLiteralValue(colTypeLowerCase, rightValueString);
+ if (invert) {
+ // column < leftValue OR column > rightValue
+ if (rightValue < leftValue) {
+ return kll.getN();
+ }
+ return Math.round(kll.getN() * (lessThanSelectivity(kll, leftValue) + greaterThanSelectivity(kll, rightValue)));
+ }
+ // if they are equal we can't handle it here, it becomes an equality predicate
+ if (Float.compare(leftValue, rightValue) != 0) {
+ return Math.round(kll.getN() * FilterSelectivityEstimator.betweenSelectivity(kll, leftValue, rightValue));
+ }
+ }
+ }
+ } catch(RuntimeException e) {
+ LOG.debug("Selectivity computation using histogram failed to parse the left ({}) or the right ({}) "
+ + "boundary value, using the generic computation strategy", leftValueString, rightValueString, e);
+ }
+
ExprNodeDesc newExpression = rewriteBetweenToIn(comparisonExpression, leftExpression, rightExpression, invert);
return evaluateExpression(stats, newExpression, aspCtx, neededCols, op, currNumRows);
@@ -968,14 +1010,13 @@ public class StatsRulesProcFactory {
private long evaluateComparator(Statistics stats, AnnotateStatsProcCtx aspCtx, ExprNodeGenericFuncDesc genFunc,
long currNumRows) {
- long numRows = currNumRows;
GenericUDF udf = genFunc.getGenericUDF();
ExprNodeColumnDesc columnDesc;
ExprNodeConstantDesc constantDesc;
boolean upperBound;
boolean closedBound;
- String boundValue = null;
+ String boundValue;
if (genFunc.getChildren().get(0) instanceof ExprNodeColumnDesc &&
genFunc.getChildren().get(1) instanceof ExprNodeConstantDesc) {
columnDesc = (ExprNodeColumnDesc) genFunc.getChildren().get(0);
@@ -1004,13 +1045,23 @@ public class StatsRulesProcFactory {
closedBound = isClosedBound(udf);
} else {
// default
- return numRows / 3;
+ return currNumRows / 3;
}
ColStatistics cs = stats.getColumnStatisticsFromColName(columnDesc.getColumn());
+ String colTypeLowerCase = columnDesc.getTypeString().toLowerCase();
+
+ if (FilterSelectivityEstimator.isHistogramAvailable(cs)) {
+ try {
+ return evaluateComparatorWithHistogram(
+ cs, currNumRows, colTypeLowerCase, boundValue, upperBound, closedBound);
+ } catch (SketchesArgumentException e) {
+ LOG.info("Sketch-based statistics estimation failed, falling back to regular estimation", e);
+ }
+ }
+
if (cs != null && cs.getRange() != null &&
cs.getRange().maxValue != null && cs.getRange().minValue != null) {
- String colTypeLowerCase = columnDesc.getTypeString().toLowerCase();
try {
if (colTypeLowerCase.equals(serdeConstants.TINYINT_TYPE_NAME)) {
byte value = Byte.parseByte(boundValue);
@@ -1018,7 +1069,7 @@ public class StatsRulesProcFactory {
byte minValue = cs.getRange().minValue.byteValue();
if (upperBound) {
if (maxValue < value || maxValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (minValue > value || minValue == value && !closedBound) {
return 0;
@@ -1026,11 +1077,11 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (value - minValue) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (value - minValue) / (maxValue - minValue)) * currNumRows);
}
} else {
if (minValue > value || minValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (maxValue < value || maxValue == value && !closedBound) {
return 0;
@@ -1038,7 +1089,7 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * currNumRows);
}
}
} else if (colTypeLowerCase.equals(serdeConstants.SMALLINT_TYPE_NAME)) {
@@ -1047,7 +1098,7 @@ public class StatsRulesProcFactory {
short minValue = cs.getRange().minValue.shortValue();
if (upperBound) {
if (maxValue < value || maxValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (minValue > value || minValue == value && !closedBound) {
return 0;
@@ -1055,11 +1106,11 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (value - minValue) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (value - minValue) / (maxValue - minValue)) * currNumRows);
}
} else {
if (minValue > value || minValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (maxValue < value || maxValue == value && !closedBound) {
return 0;
@@ -1067,7 +1118,7 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * currNumRows);
}
}
} else if (colTypeLowerCase.equals(serdeConstants.INT_TYPE_NAME) ||
@@ -1084,7 +1135,7 @@ public class StatsRulesProcFactory {
int minValue = cs.getRange().minValue.intValue();
if (upperBound) {
if (maxValue < value || maxValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (minValue > value || minValue == value && !closedBound) {
return 0;
@@ -1092,11 +1143,11 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (value - minValue) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (value - minValue) / (maxValue - minValue)) * currNumRows);
}
} else {
if (minValue > value || minValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (maxValue < value || maxValue == value && !closedBound) {
return 0;
@@ -1104,7 +1155,7 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * currNumRows);
}
}
} else if (colTypeLowerCase.equals(serdeConstants.BIGINT_TYPE_NAME) ||
@@ -1120,7 +1171,7 @@ public class StatsRulesProcFactory {
long minValue = cs.getRange().minValue.longValue();
if (upperBound) {
if (maxValue < value || maxValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (minValue > value || minValue == value && !closedBound) {
return 0;
@@ -1128,11 +1179,11 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (value - minValue) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (value - minValue) / (maxValue - minValue)) * currNumRows);
}
} else {
if (minValue > value || minValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (maxValue < value || maxValue == value && !closedBound) {
return 0;
@@ -1140,7 +1191,7 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * currNumRows);
}
}
} else if (colTypeLowerCase.equals(serdeConstants.FLOAT_TYPE_NAME)) {
@@ -1149,7 +1200,7 @@ public class StatsRulesProcFactory {
float minValue = cs.getRange().minValue.floatValue();
if (upperBound) {
if (maxValue < value || maxValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (minValue > value || minValue == value && !closedBound) {
return 0;
@@ -1157,11 +1208,11 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (value - minValue) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (value - minValue) / (maxValue - minValue)) * currNumRows);
}
} else {
if (minValue > value || minValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (maxValue < value || maxValue == value && !closedBound) {
return 0;
@@ -1169,7 +1220,7 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * numRows);
+ return Math.round(((double) (maxValue - value) / (maxValue - minValue)) * currNumRows);
}
}
} else if (colTypeLowerCase.equals(serdeConstants.DOUBLE_TYPE_NAME)) {
@@ -1178,7 +1229,7 @@ public class StatsRulesProcFactory {
double minValue = cs.getRange().minValue.doubleValue();
if (upperBound) {
if (maxValue < value || maxValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (minValue > value || minValue == value && !closedBound) {
return 0;
@@ -1186,11 +1237,11 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((value - minValue) / (maxValue - minValue)) * numRows);
+ return Math.round(((value - minValue) / (maxValue - minValue)) * currNumRows);
}
} else {
if (minValue > value || minValue == value && closedBound) {
- return numRows;
+ return currNumRows;
}
if (maxValue < value || maxValue == value && !closedBound) {
return 0;
@@ -1198,7 +1249,7 @@ public class StatsRulesProcFactory {
if (aspCtx.isUniformWithinRange()) {
// Assuming uniform distribution, we can use the range to calculate
// new estimate for the number of rows
- return Math.round(((maxValue - value) / (maxValue - minValue)) * numRows);
+ return Math.round(((maxValue - value) / (maxValue - minValue)) * currNumRows);
}
}
} else if (colTypeLowerCase.startsWith(serdeConstants.DECIMAL_TYPE_NAME)) {
@@ -1209,7 +1260,7 @@ public class StatsRulesProcFactory {
int maxComparison = value.compareTo(maxValue);
if (upperBound) {
if (maxComparison > 0 || maxComparison == 0 && closedBound) {
- return numRows;
+ return currNumRows;
}
if (minComparison < 0 || minComparison == 0 && !closedBound) {
return 0;
@@ -1219,12 +1270,12 @@ public class StatsRulesProcFactory {
// new estimate for the number of rows
return Math.round(
((value.subtract(minValue)).divide(maxValue.subtract(minValue), RoundingMode.UP))
- .multiply(BigDecimal.valueOf(numRows))
+ .multiply(BigDecimal.valueOf(currNumRows))
.doubleValue());
}
} else {
if (minComparison < 0 || minComparison == 0 && closedBound) {
- return numRows;
+ return currNumRows;
}
if (maxComparison > 0 || maxComparison == 0 && !closedBound) {
return 0;
@@ -1234,17 +1285,70 @@ public class StatsRulesProcFactory {
// new estimate for the number of rows
return Math.round(
((maxValue.subtract(value)).divide(maxValue.subtract(minValue), RoundingMode.UP))
- .multiply(BigDecimal.valueOf(numRows))
+ .multiply(BigDecimal.valueOf(currNumRows))
.doubleValue());
}
}
}
} catch (NumberFormatException nfe) {
- return numRows / 3;
+ return currNumRows / 3;
}
}
// default
- return numRows / 3;
+ return currNumRows / 3;
+ }
+
+ private long evaluateComparatorWithHistogram(ColStatistics cs, long currNumRows, String colTypeLowerCase,
+ String boundValue, boolean upperBound, boolean closedBound) {
+ final KllFloatsSketch kll = KllFloatsSketch.heapify(Memory.wrap(cs.getHistogram()));
+
+ if (kll.getN() == 0) {
+ return 0;
+ }
+
+ try {
+ final float value = extractFloatFromLiteralValue(colTypeLowerCase, boundValue);
+
+ // kll ignores null values (i.e., kll.getN() + numNulls = currNumRows), we therefore need to use kll.getN()
+ // instead of currNumRows since the CDF is expressed as a fraction of kll.getN(), not currNumRows
+ if (upperBound) {
+ return Math.round(kll.getN() * (closedBound ?
+ lessThanOrEqualSelectivity(kll, value) : lessThanSelectivity(kll, value)));
+ } else {
+ return Math.round(kll.getN() * (closedBound ?
+ greaterThanOrEqualSelectivity(kll, value) : greaterThanSelectivity(kll, value)));
+ }
+ } catch (RuntimeException e) {
+ LOG.debug("Selectivity computation using histogram failed to parse the boundary value ({}), "
+ + ", using the generic computation strategy", boundValue, e);
+ return currNumRows / 3;
+ }
+ }
+
+ @VisibleForTesting
+ protected static float extractFloatFromLiteralValue(String colTypeLowerCase, String value) {
+ if (colTypeLowerCase.equals(serdeConstants.TINYINT_TYPE_NAME)) {
+ return Byte.parseByte(value);
+ } else if (colTypeLowerCase.equals(serdeConstants.SMALLINT_TYPE_NAME)) {
+ return Short.parseShort(value);
+ } else if (colTypeLowerCase.equals(serdeConstants.INT_TYPE_NAME)) {
+ return Integer.parseInt(value);
+ } else if (colTypeLowerCase.equals(serdeConstants.BIGINT_TYPE_NAME)) {
+ return Long.parseLong(value);
+ } else if (colTypeLowerCase.equals(serdeConstants.FLOAT_TYPE_NAME)) {
+ return Float.parseFloat(value);
+ } else if (colTypeLowerCase.equals(serdeConstants.DOUBLE_TYPE_NAME)) {
+ return (float) Double.parseDouble(value);
+ } else if (colTypeLowerCase.startsWith(serdeConstants.DECIMAL_TYPE_NAME)) {
+ return new BigDecimal(value).floatValue();
+ } else if (colTypeLowerCase.startsWith(serdeConstants.TIMESTAMP_TYPE_NAME)) {
+ return Timestamp.valueOf(value).toEpochSecond();
+ } else if (colTypeLowerCase.startsWith(serdeConstants.DATE_TYPE_NAME)) {
+ return Date.valueOf(value).toEpochSecond();
+ } else {
+ throw new IllegalStateException(
+ "Unsupported type for comparator selectivity evaluation using histogram: " + colTypeLowerCase);
+ }
}
private boolean isClosedBound(GenericUDF udf) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
index 6a00131b254..5d4768000a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.parse;
import static org.apache.hadoop.hive.ql.metadata.HiveUtils.unparseIdentifier;
import com.google.common.base.Preconditions;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -311,13 +310,15 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
Preconditions.checkArgument(typeInfo.getCategory() == Category.PRIMITIVE);
ColumnStatsType columnStatsType =
ColumnStatsType.getColumnStatsType((PrimitiveTypeInfo) typeInfo);
+ List<ColumnStatsField> columnStatsFields = columnStatsType.getColumnStats();
+ columnStatsFields = ColumnStatsType.removeDisabledStatistics(conf, columnStatsFields);
// The first column is always the type
// The rest of columns will depend on the type itself
- for (int i = 0; i < columnStatsType.getColumnStats().size(); i++) {
+ for (int i = 0; i < columnStatsFields.size(); i++) {
+ ColumnStatsField columnStatsField = columnStatsFields.get(i);
if (i > 0) {
rewrittenQueryBuilder.append(", ");
}
- ColumnStatsField columnStatsField = columnStatsType.getColumnStats().get(i);
appendStatsField(rewrittenQueryBuilder, conf, columnStatsField, columnStatsType,
columnName, pos);
}
@@ -351,6 +352,9 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
case BITVECTOR:
appendBitVector(rewrittenQueryBuilder, conf, columnName, pos);
break;
+ case KLL_SKETCH:
+ appendKllSketch(rewrittenQueryBuilder, conf, columnName, columnStatsType, pos);
+ break;
case MAX_LENGTH:
appendMaxLength(rewrittenQueryBuilder, conf, columnName, pos);
break;
@@ -455,6 +459,13 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
.append(unparseIdentifier(ColumnStatsField.BITVECTOR.getFieldName() + pos, conf));
}
+ private static void appendKllSketch(StringBuilder rewrittenQueryBuilder, HiveConf conf,
+ String columnName, ColumnStatsType columnStatsType, int pos) throws SemanticException {
+ appendKllSketch(rewrittenQueryBuilder, conf, columnName, columnStatsType);
+ rewrittenQueryBuilder.append(" AS ")
+ .append(unparseIdentifier(ColumnStatsField.KLL_SKETCH.getFieldName() + pos, conf));
+ }
+
private static void appendBitVector(StringBuilder rewrittenQueryBuilder, HiveConf conf,
String columnName) throws SemanticException {
String func = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_STATS_NDV_ALGO).toLowerCase();
@@ -480,6 +491,33 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
}
}
+ private static void appendKllSketch(StringBuilder rewrittenQueryBuilder, HiveConf conf, String columnName,
+ ColumnStatsType columnStatsType) throws SemanticException {
+ int k;
+ try {
+ k = HiveStatsUtils.getKParamForKllSketch(conf);
+ } catch (Exception e) {
+ throw new SemanticException(e.getMessage());
+ }
+
+ boolean isDateFamily = columnStatsType.equals(ColumnStatsType.DATE) ||
+ columnStatsType.equals(ColumnStatsType.TIMESTAMP);
+
+ if (isDateFamily) {
+ rewrittenQueryBuilder.append("ds_kll_sketch(cast(unix_timestamp(")
+ .append(columnName)
+ .append(") as float)");
+ } else {
+ // add cast to float to make sure it works for other numeric types
+ rewrittenQueryBuilder.append("ds_kll_sketch(cast(")
+ .append(columnName)
+ .append(" as float)");
+ }
+ rewrittenQueryBuilder.append(", ")
+ .append(k)
+ .append(")");
+ }
+
private static void appendCountTrues(StringBuilder rewrittenQueryBuilder, HiveConf conf,
String columnName, int pos) {
rewrittenQueryBuilder.append("CAST(count(CASE WHEN ")
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java
index 7142a05803d..75cdba5abd8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java
@@ -32,6 +32,7 @@ public class ColStatistics {
private boolean isEstimated;
private boolean isFilteredColumn;
private byte[] bitVectors;
+ private byte[] histogram;
public ColStatistics(String colName, String colType) {
this.setColumnName(colName);
@@ -120,6 +121,14 @@ public class ColStatistics {
this.bitVectors = bitVectors;
}
+ public byte[] getHistogram() {
+ return histogram;
+ }
+
+ public void setHistogram(byte[] histogram) {
+ this.histogram = histogram;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -157,6 +166,8 @@ public class ColStatistics {
clone.setNumNulls(numNulls);
clone.setNumTrues(numTrues);
clone.setNumFalses(numFalses);
+ clone.setBitVectors(bitVectors);
+ clone.setHistogram(histogram);
clone.setPrimaryKey(isPrimaryKey);
clone.setIsEstimated(isEstimated);
clone.setIsFilteredColumn(isFilteredColumn);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java
index 49aa28871b0..95d4b439d16 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java
@@ -23,7 +23,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hive.common.HiveStatsUtils;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -35,7 +37,6 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.exec.FetchOperator;
import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -119,6 +120,7 @@ public class ColStatsProcessor implements IStatsProcessor {
String columnType = colType.get(i);
PrimitiveTypeInfo typeInfo = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(columnType);
List<ColumnStatsField> columnStatsFields = ColumnStatsType.getColumnStats(typeInfo);
+ columnStatsFields = ColumnStatsType.removeDisabledStatistics(conf, columnStatsFields);
try {
ColumnStatisticsObj statObj = ColumnStatisticsObjTranslator.readHiveColumnStatistics(
columnName, columnType, columnStatsFields, pos, fields, values);
@@ -240,6 +242,7 @@ public class ColStatsProcessor implements IStatsProcessor {
MAX("max"),
NDV("numdistinctvalues"),
BITVECTOR("ndvbitvector"),
+ KLL_SKETCH("kllsketch"),
MAX_LENGTH("maxlength"),
AVG_LENGTH("avglength");
@@ -274,7 +277,8 @@ public class ColStatsProcessor implements IStatsProcessor {
ColumnStatsField.MAX,
ColumnStatsField.COUNT_NULLS,
ColumnStatsField.NDV,
- ColumnStatsField.BITVECTOR)),
+ ColumnStatsField.BITVECTOR,
+ ColumnStatsField.KLL_SKETCH)),
DOUBLE(
ImmutableList.of(
ColumnStatsField.COLUMN_STATS_TYPE,
@@ -282,7 +286,8 @@ public class ColStatsProcessor implements IStatsProcessor {
ColumnStatsField.MAX,
ColumnStatsField.COUNT_NULLS,
ColumnStatsField.NDV,
- ColumnStatsField.BITVECTOR)),
+ ColumnStatsField.BITVECTOR,
+ ColumnStatsField.KLL_SKETCH)),
STRING(
ImmutableList.of(
ColumnStatsField.COLUMN_STATS_TYPE,
@@ -304,7 +309,8 @@ public class ColStatsProcessor implements IStatsProcessor {
ColumnStatsField.MAX,
ColumnStatsField.COUNT_NULLS,
ColumnStatsField.NDV,
- ColumnStatsField.BITVECTOR)),
+ ColumnStatsField.BITVECTOR,
+ ColumnStatsField.KLL_SKETCH)),
DATE(
ImmutableList.of(
ColumnStatsField.COLUMN_STATS_TYPE,
@@ -312,7 +318,8 @@ public class ColStatsProcessor implements IStatsProcessor {
ColumnStatsField.MAX,
ColumnStatsField.COUNT_NULLS,
ColumnStatsField.NDV,
- ColumnStatsField.BITVECTOR)),
+ ColumnStatsField.BITVECTOR,
+ ColumnStatsField.KLL_SKETCH)),
TIMESTAMP(
ImmutableList.of(
ColumnStatsField.COLUMN_STATS_TYPE,
@@ -320,7 +327,8 @@ public class ColStatsProcessor implements IStatsProcessor {
ColumnStatsField.MAX,
ColumnStatsField.COUNT_NULLS,
ColumnStatsField.NDV,
- ColumnStatsField.BITVECTOR));
+ ColumnStatsField.BITVECTOR,
+ ColumnStatsField.KLL_SKETCH));
private final List<ColumnStatsField> columnStats;
@@ -370,5 +378,13 @@ public class ColStatsProcessor implements IStatsProcessor {
return getColumnStatsType(typeInfo).getColumnStats();
}
+ public static List<ColumnStatsField> removeDisabledStatistics(HiveConf conf, List<ColumnStatsField> columnStatsFields) {
+ if (!HiveStatsUtils.computeHistograms(conf)) {
+ return columnStatsFields.stream()
+ .filter(f -> f != ColumnStatsField.KLL_SKETCH)
+ .collect(Collectors.toList());
+ }
+ return columnStatsFields;
+ }
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
index 799bc386e59..6bffa52eea9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
@@ -128,6 +128,11 @@ public class ColumnStatisticsObjTranslator {
byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o);
statsObj.getStatsData().getDoubleStats().setBitVectors(buf);
break;
+ case KLL_SKETCH:
+ PrimitiveObjectInspector poi2 = (PrimitiveObjectInspector) oi;
+ byte[] buf2 = ((BinaryObjectInspector) poi2).getPrimitiveJavaObject(o);
+ statsObj.getStatsData().getDoubleStats().setHistogram(buf2);
+ break;
default:
throw new RuntimeException("Unsupported column stat for DOUBLE : " + csf);
}
@@ -157,6 +162,11 @@ public class ColumnStatisticsObjTranslator {
byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o);
statsObj.getStatsData().getDecimalStats().setBitVectors(buf);
break;
+ case KLL_SKETCH:
+ PrimitiveObjectInspector poi2 = (PrimitiveObjectInspector) oi;
+ byte[] buf2 = ((BinaryObjectInspector) poi2).getPrimitiveJavaObject(o);
+ statsObj.getStatsData().getDecimalStats().setHistogram(buf2);
+ break;
default:
throw new RuntimeException("Unsupported column stat for DECIMAL : " + csf);
}
@@ -190,6 +200,11 @@ public class ColumnStatisticsObjTranslator {
byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o);
statsObj.getStatsData().getLongStats().setBitVectors(buf);
break;
+ case KLL_SKETCH:
+ PrimitiveObjectInspector poi2 = (PrimitiveObjectInspector) oi;
+ byte[] buf2 = ((BinaryObjectInspector) poi2).getPrimitiveJavaObject(o);
+ statsObj.getStatsData().getLongStats().setHistogram(buf2);
+ break;
default:
throw new RuntimeException("Unsupported column stat for LONG : " + csf);
}
@@ -268,6 +283,11 @@ public class ColumnStatisticsObjTranslator {
byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o);
statsObj.getStatsData().getDateStats().setBitVectors(buf);
break;
+ case KLL_SKETCH:
+ PrimitiveObjectInspector poi2 = (PrimitiveObjectInspector) oi;
+ byte[] buf2 = ((BinaryObjectInspector) poi2).getPrimitiveJavaObject(o);
+ statsObj.getStatsData().getDateStats().setHistogram(buf2);
+ break;
default:
throw new RuntimeException("Unsupported column stat for DATE : " + csf);
}
@@ -297,6 +317,11 @@ public class ColumnStatisticsObjTranslator {
byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o);
statsObj.getStatsData().getTimestampStats().setBitVectors(buf);
break;
+ case KLL_SKETCH:
+ PrimitiveObjectInspector poi2 = (PrimitiveObjectInspector) oi;
+ byte[] buf2 = ((BinaryObjectInspector) poi2).getPrimitiveJavaObject(o);
+ statsObj.getStatsData().getTimestampStats().setHistogram(buf2);
+ break;
default:
throw new RuntimeException("Unsupported column stat for TIMESTAMP : " + csf);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index babb5dbc330..9c1926a747e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -864,24 +864,28 @@ public class StatsUtils {
cs.setAvgColLen(JavaDataModel.get().primitive1());
cs.setRange(csd.getLongStats().getLowValue(), csd.getLongStats().getHighValue());
cs.setBitVectors(csd.getLongStats().getBitVectors());
+ cs.setHistogram(csd.getLongStats().getHistogram());
} else if (colTypeLowerCase.equals(serdeConstants.BIGINT_TYPE_NAME)) {
cs.setCountDistint(csd.getLongStats().getNumDVs());
cs.setNumNulls(csd.getLongStats().getNumNulls());
cs.setAvgColLen(JavaDataModel.get().primitive2());
cs.setRange(csd.getLongStats().getLowValue(), csd.getLongStats().getHighValue());
cs.setBitVectors(csd.getLongStats().getBitVectors());
+ cs.setHistogram(csd.getLongStats().getHistogram());
} else if (colTypeLowerCase.equals(serdeConstants.FLOAT_TYPE_NAME)) {
cs.setCountDistint(csd.getDoubleStats().getNumDVs());
cs.setNumNulls(csd.getDoubleStats().getNumNulls());
cs.setAvgColLen(JavaDataModel.get().primitive1());
cs.setRange(csd.getDoubleStats().getLowValue(), csd.getDoubleStats().getHighValue());
cs.setBitVectors(csd.getDoubleStats().getBitVectors());
+ cs.setHistogram(csd.getDoubleStats().getHistogram());
} else if (colTypeLowerCase.equals(serdeConstants.DOUBLE_TYPE_NAME)) {
cs.setCountDistint(csd.getDoubleStats().getNumDVs());
cs.setNumNulls(csd.getDoubleStats().getNumNulls());
cs.setAvgColLen(JavaDataModel.get().primitive2());
cs.setRange(csd.getDoubleStats().getLowValue(), csd.getDoubleStats().getHighValue());
cs.setBitVectors(csd.getDoubleStats().getBitVectors());
+ cs.setHistogram(csd.getDoubleStats().getHistogram());
} else if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME)
|| colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)
|| colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME)) {
@@ -910,6 +914,7 @@ public class StatsUtils {
Long highVal = (csd.getTimestampStats().getHighValue() != null) ? csd.getTimestampStats().getHighValue()
.getSecondsSinceEpoch() : null;
cs.setRange(lowVal, highVal);
+ cs.setHistogram(csd.getTimestampStats().getHistogram());
} else if (colTypeLowerCase.equals(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME)) {
cs.setAvgColLen(JavaDataModel.get().lengthOfTimestamp());
} else if (colTypeLowerCase.startsWith(serdeConstants.DECIMAL_TYPE_NAME)) {
@@ -930,6 +935,7 @@ public class StatsUtils {
}
}
cs.setBitVectors(csd.getDecimalStats().getBitVectors());
+ cs.setHistogram(csd.getDecimalStats().getHistogram());
} else if (colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) {
cs.setAvgColLen(JavaDataModel.get().lengthOfDate());
cs.setNumNulls(csd.getDateStats().getNumNulls());
@@ -939,6 +945,7 @@ public class StatsUtils {
.getDaysSinceEpoch() : null;
cs.setRange(lowVal, highVal);
cs.setBitVectors(csd.getDateStats().getBitVectors());
+ cs.setHistogram(csd.getDateStats().getHistogram());
} else {
// Columns statistics for complex datatypes are not supported yet
return null;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/stats/TestFilterSelectivityEstimator.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/stats/TestFilterSelectivityEstimator.java
new file mode 100644
index 00000000000..ade4f89d0e6
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/stats/TestFilterSelectivityEstimator.java
@@ -0,0 +1,459 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.stats;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptSchema;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.datasketches.kll.KllFloatsSketch;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.StatisticsTestUtils;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
+import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
+import org.apache.hadoop.hive.ql.parse.CalcitePlanner;
+import org.apache.hadoop.hive.ql.plan.ColStatistics;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Collections;
+
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.betweenSelectivity;
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.greaterThanOrEqualSelectivity;
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.greaterThanSelectivity;
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.isHistogramAvailable;
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.lessThanOrEqualSelectivity;
+import static org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator.lessThanSelectivity;
+import static org.mockito.Mockito.doReturn;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestFilterSelectivityEstimator {
+
+ private static final float[] VALUES = { 1, 2, 2, 2, 2, 2, 2, 2, 3, 4, 5, 6, 7 };
+ private static final KllFloatsSketch KLL = StatisticsTestUtils.createKll(VALUES);
+ private static final float DELTA = Float.MIN_VALUE;
+ private static final RexBuilder REX_BUILDER = new RexBuilder(new JavaTypeFactoryImpl(new HiveTypeSystemImpl()));
+ private static final RelDataTypeFactory TYPE_FACTORY = REX_BUILDER.getTypeFactory();
+
+ private static RelOptCluster relOptCluster;
+
+ private static RexNode intMinus1;
+ private static RexNode int0;
+ private static RexNode int1;
+ private static RexNode int2;
+ private static RexNode int3;
+ private static RexNode int4;
+ private static RexNode int5;
+ private static RexNode int7;
+ private static RexNode int8;
+ private static RexNode int10;
+ private static RexNode int11;
+ private static RelDataType tableType;
+ private static RexNode inputRef0;
+ private static ColStatistics stats;
+
+ @Mock
+ private RelOptSchema schemaMock;
+ @Mock
+ private RelOptHiveTable tableMock;
+ @Mock
+ private RelMetadataQuery mq;
+
+ private HiveTableScan tableScan;
+ private RelNode scan;
+
+ @BeforeClass
+ public static void beforeClass() {
+ RelDataType integerType = TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER);
+ intMinus1 = REX_BUILDER.makeLiteral(-1, integerType, true);
+ int0 = REX_BUILDER.makeLiteral(0, integerType, true);
+ int1 = REX_BUILDER.makeLiteral(1, integerType, true);
+ int2 = REX_BUILDER.makeLiteral(2, integerType, true);
+ int3 = REX_BUILDER.makeLiteral(3, integerType, true);
+ int4 = REX_BUILDER.makeLiteral(4, integerType, true);
+ int5 = REX_BUILDER.makeLiteral(5, integerType, true);
+ int7 = REX_BUILDER.makeLiteral(7, integerType, true);
+ int8 = REX_BUILDER.makeLiteral(8, integerType, true);
+ int10 = REX_BUILDER.makeLiteral(10, integerType, true);
+ int11 = REX_BUILDER.makeLiteral(11, integerType, true);
+ tableType = TYPE_FACTORY.createStructType(ImmutableList.of(integerType), ImmutableList.of("f1"));
+
+ RelOptPlanner planner = CalcitePlanner.createPlanner(new HiveConf());
+ relOptCluster = RelOptCluster.create(planner, REX_BUILDER);
+
+ stats = new ColStatistics();
+ stats.setHistogram(KLL.toByteArray());
+ }
+
+ @Before
+ public void before() {
+ doReturn(tableType).when(tableMock).getRowType();
+ doReturn((double) VALUES.length).when(tableMock).getRowCount();
+
+ RelBuilder relBuilder = HiveRelFactories.HIVE_BUILDER.create(relOptCluster, schemaMock);
+ tableScan = new HiveTableScan(relOptCluster, relOptCluster.traitSetOf(HiveRelNode.CONVENTION),
+ tableMock, "table", null, false, false);
+ scan = relBuilder.push(tableScan).build();
+ inputRef0 = REX_BUILDER.makeInputRef(scan, 0);
+ }
+
+ @Test
+ public void testIsHistogramAvailableWhenAvailable() {
+ ColStatistics colStatistics = new ColStatistics();
+ colStatistics.setHistogram(KLL.toByteArray());
+ Assert.assertTrue(isHistogramAvailable(colStatistics));
+ }
+
+ @Test
+ public void testIsHistogramAvailableWhenNullStatistics() {
+ Assert.assertFalse(isHistogramAvailable(null));
+ }
+
+ @Test
+ public void testIsHistogramAvailableWhenNullHistogram() {
+ Assert.assertFalse(isHistogramAvailable(new ColStatistics()));
+ }
+
+ @Test
+ public void testIsHistogramAvailableWhenEmptyArray() {
+ ColStatistics colStatistics = new ColStatistics();
+ colStatistics.setHistogram(new byte[0]);
+ Assert.assertFalse(isHistogramAvailable(colStatistics));
+ }
+
+ @Test
+ public void testLessThanSelectivity() {
+ Assert.assertEquals(0.6153846153846154, lessThanSelectivity(KLL, 3), DELTA);
+ }
+
+ @Test
+ public void testLessThanSelectivityWhenLowerThanMin() {
+ Assert.assertEquals(0, lessThanSelectivity(KLL, 0), DELTA);
+ }
+
+ @Test
+ public void testLessThanSelectivityWhenHigherThanMax() {
+ Assert.assertEquals(1, lessThanSelectivity(KLL, 10), DELTA);
+ }
+
+ @Test
+ public void testLessThanOrEqualSelectivity() {
+ Assert.assertEquals(0.6923076923076923, lessThanOrEqualSelectivity(KLL, 3), DELTA);
+ }
+
+ @Test
+ public void testLessThanOrEqualSelectivityWhenLowerThanMin() {
+ Assert.assertEquals(0, lessThanOrEqualSelectivity(KLL, 0), DELTA);
+ }
+
+ @Test
+ public void testLessThanOrEqualSelectivityWhenHigherThanMax() {
+ Assert.assertEquals(1, lessThanOrEqualSelectivity(KLL, 10), DELTA);
+ }
+
+ @Test
+ public void testGreaterThanSelectivity() {
+ Assert.assertEquals(0.3076923076923077, greaterThanSelectivity(KLL, 3), DELTA);
+ }
+
+ @Test
+ public void testGreaterThanSelectivityWhenLowerThanMin() {
+ Assert.assertEquals(1, greaterThanSelectivity(KLL, 0), DELTA);
+ }
+
+ @Test
+ public void testGreaterThanSelectivityWhenHigherThanMax() {
+ Assert.assertEquals(0, greaterThanSelectivity(KLL, 10), DELTA);
+ }
+
+ @Test
+ public void testGreaterThanOrEqualSelectivity() {
+ Assert.assertEquals(0.3846153846153846, greaterThanOrEqualSelectivity(KLL, 3), DELTA);
+ }
+
+ @Test
+ public void testGreaterThanOrEqualSelectivityWhenLowerThanMin() {
+ Assert.assertEquals(1, greaterThanOrEqualSelectivity(KLL, 0), DELTA);
+ }
+
+ @Test
+ public void testGreaterThanOrEqualSelectivityWhenHigherThanMax() {
+ Assert.assertEquals(0, greaterThanOrEqualSelectivity(KLL, 10), DELTA);
+ }
+
+ @Test
+ public void testBetweenSelectivity() {
+ Assert.assertEquals(0.6923076923076923, betweenSelectivity(KLL, 1, 3), DELTA);
+ }
+
+ @Test
+ public void testBetweenSelectivityFromMinToMax() {
+ Assert.assertEquals(1, betweenSelectivity(KLL, 1, 7), DELTA);
+ }
+
+ @Test
+ public void testBetweenSelectivityFromLowerThanMinToHigherThanMax() {
+ Assert.assertEquals(1, betweenSelectivity(KLL, 0, 8), DELTA);
+ }
+
+ @Test
+ public void testBetweenSelectivityLeftLowerThanMin() {
+ Assert.assertEquals(0.6923076923076923, betweenSelectivity(KLL, 0, 3), DELTA);
+ }
+
+ @Test
+ public void testBetweenSelectivityRightLowerThanMin() {
+ Assert.assertEquals(0, betweenSelectivity(KLL, -1, 0), DELTA);
+ }
+
+ @Test
+ public void testBetweenSelectivityLeftHigherThanMax() {
+ Assert.assertEquals(0, betweenSelectivity(KLL, 10, 11), DELTA);
+ }
+
+ @Test
+ public void testBetweenSelectivityLeftLowerThanRight() {
+ Assert.assertEquals(0, betweenSelectivity(KLL, 4, 2), DELTA);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBetweenSelectivityLeftEqualsRight_KO() {
+ betweenSelectivity(KLL, 2, 2);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityWhenNoStats() {
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.LESS_THAN, inputRef0, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ // defaults to 1/3 in the absence of stats
+ Assert.assertEquals(0.3333333333333333, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityLessThan() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.LESS_THAN, inputRef0, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0.6153846153846154, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityLessThanWhenLowerThanMin() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.LESS_THAN, inputRef0, int0);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityLessThanWhenHigherThanMax() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.LESS_THAN, inputRef0, int10);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(1, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityLessThanOrEqual() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, inputRef0, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0.6923076923076923, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityLessThanOrEqualWhenLowerThanMin() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, inputRef0, int0);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityLessThanOrEqualWhenHigherThanMax() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, inputRef0, int10);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(1, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityGreaterThan() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.GREATER_THAN, inputRef0, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0.3076923076923077, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityGreaterThanWhenLowerThanMin() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.GREATER_THAN, inputRef0, int0);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(1, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityGreaterThanWhenHigherThanMax() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.GREATER_THAN, inputRef0, int10);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityGreaterThanOrEqual() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, inputRef0, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0.38461538461538464, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityGreaterThanOrEqualWhenLowerThanMin() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, inputRef0, int0);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(1, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityGreaterThanOrEqualWhenHigherThanMax() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, inputRef0, int10);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityBetween() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.BETWEEN, inputRef0, int1, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0.6923076923076923, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityBetweenFromMinToMax() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.BETWEEN, inputRef0, int1, int7);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(1, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityBetweenFromLowerThanMinToHigherThanMax() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.BETWEEN, inputRef0, int0, int8);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(1, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityBetweenLeftLowerThanMin() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.BETWEEN, inputRef0, int0, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0.6923076923076923, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityBetweenRightLowerThanMin() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.BETWEEN, inputRef0, intMinus1, int0);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityBetweenLeftHigherThanMax() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.BETWEEN, inputRef0, int10, int11);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityBetweenLeftLowerThanRight() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.BETWEEN, inputRef0, int4, int2);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityBetweenLeftEqualsRight() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ doReturn(10.0).when(mq).getDistinctRowCount(scan, ImmutableBitSet.of(0), REX_BUILDER.makeLiteral(true));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.BETWEEN, inputRef0, int3, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ // this is what FilterSelectivityEstimator returns for a generic "function" based on NDV values, in this case 1 / 10
+ Assert.assertEquals(0.1, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityNotBetween() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.NOT_BETWEEN, inputRef0, int3, int5);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0.7692307692307693, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityNotBetweenLowerThanMinHigherThanMax() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.NOT_BETWEEN, inputRef0, int0, int10);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(0, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityNotBetweenRightLowerThanLeft() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.NOT_BETWEEN, inputRef0, int5, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(1, estimator.estimateSelectivity(filter), DELTA);
+ }
+
+ @Test
+ public void testComputeRangePredicateSelectivityNotBetweenLeftEqualsRight() {
+ doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+ RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.NOT_BETWEEN, inputRef0, int3, int3);
+ FilterSelectivityEstimator estimator = new FilterSelectivityEstimator(scan, mq);
+ Assert.assertEquals(1, estimator.estimateSelectivity(filter), DELTA);
+ }
+}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/stats/annotation/TestStatsRulesProcFactory.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/stats/annotation/TestStatsRulesProcFactory.java
new file mode 100644
index 00000000000..1290a54c1f3
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/stats/annotation/TestStatsRulesProcFactory.java
@@ -0,0 +1,634 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.stats.annotation;
+
+import org.apache.datasketches.kll.KllFloatsSketch;
+import org.apache.hadoop.hive.common.ndv.fm.FMSketch;
+import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.metastore.StatisticsTestUtils;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ColStatistics;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.Statistics;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.apache.hadoop.hive.ql.optimizer.stats.annotation.StatsRulesProcFactory.FilterStatsRule.extractFloatFromLiteralValue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+
+public class TestStatsRulesProcFactory {
+
+ private final static String COL_NAME = "col1";
+ private final static ExprNodeDesc COL_EXPR = new ExprNodeColumnDesc(
+ TypeInfoFactory.intTypeInfo, COL_NAME, "table", false);
+
+ private final static AnnotateStatsProcCtx STATS_PROC_CTX = new AnnotateStatsProcCtx(null);
+
+ private final static long[] VALUES = { 1L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 3L, 4L, 5L, 6L, 7L };
+
+ @Test
+ public void testComparisonRowCountZeroNonNullValues() throws SemanticException {
+ long numNulls = 2;
+ long[] values = {};
+ Statistics stats = createStatistics(values, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(3)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountInvalidKll() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ stats.getColumnStats().get(0).setHistogram(null);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(3)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ // when no KLL it defaults to 1/3 of the number of rows
+ assertEquals((VALUES.length + numNulls) / 3, numRows);
+
+ // empty KLL array is not valid either
+ stats.getColumnStats().get(0).setHistogram(new byte[0]);
+ numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ // when no KLL it defaults to 1/3 of the number of rows
+ assertEquals((VALUES.length + numNulls) / 3, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountLessThan() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(3)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(8, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountLessThanMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(1)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountLessThanBelowMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(0)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountLessThanMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(7)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(12, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountLessThanAboveMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(8)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(13, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrLessThan() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(3)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(9, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrLessThanMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(1)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(1, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrLessThanBelowMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(0)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrLessThanMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(7)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(13, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrLessThanAboveMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(8)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(13, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountGreaterThan() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(5)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(2, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountGreaterThanMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(1)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(12, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountGreaterThanBelowMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(0)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(13, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountGreaterThanMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(7)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountGreaterThanAboveMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(8)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrGreaterThan() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(5)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(3, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrGreaterThanMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(1)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(13, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrGreaterThanBelowMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(0)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(13, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrGreaterThanMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(7)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(1, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrGreaterThanBeyondMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(8)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrLessThanWhenMinEqualMax() throws SemanticException {
+ long[] values = { 1L, 1L };
+ long numNulls = 2;
+ Statistics stats = createStatistics(values, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(1)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, values.length + numNulls);
+
+ assertEquals(2, numRows);
+ }
+
+ @Test
+ public void testComparisonRowCountEqualOrGreaterThanWhenMinEqualMax() throws SemanticException {
+ long[] values = { 1L, 1L };
+ long numNulls = 2;
+ Statistics stats = createStatistics(values, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(1)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, values.length + numNulls);
+
+ assertEquals(2, numRows);
+ }
+
+ @Test
+ public void testBetween() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.FALSE),
+ COL_EXPR, createExprNodeConstantDesc(3), createExprNodeConstantDesc(4)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(2, numRows);
+ }
+
+ @Test
+ public void testLiteralExtraction() {
+ final double DELTA = 1e-5;
+
+ assertEquals((float) 100,
+ extractFloatFromLiteralValue("int", "100"), DELTA);
+ assertEquals((float) 1,
+ extractFloatFromLiteralValue("smallint", "1"), DELTA);
+ assertEquals((float) 1,
+ extractFloatFromLiteralValue("tinyint", "1"), DELTA);
+ assertEquals((float) 10000000000000L,
+ extractFloatFromLiteralValue("bigint", "10000000000000"), DELTA);
+ assertEquals((float) 15.2,
+ extractFloatFromLiteralValue("decimal(5, 2)", "15.2"), DELTA);
+ assertEquals(15.0002f,
+ extractFloatFromLiteralValue("float", "15.0002"), DELTA);
+ assertEquals((float) 1512.2222222d,
+ extractFloatFromLiteralValue("double", "1512.2222222d"), DELTA);
+ assertEquals((float) Date.valueOf("2022-01-05").toEpochSecond(),
+ extractFloatFromLiteralValue("date", "2022-01-05"), DELTA);
+ assertEquals((float) Date.valueOf("2022-1-5").toEpochSecond(),
+ extractFloatFromLiteralValue("date", "2022-1-5"), DELTA);
+ assertEquals((float) Timestamp.valueOf("2022-01-05 00:00:00").toEpochSecond(),
+ extractFloatFromLiteralValue("timestamp", "2022-01-05 00:00:00"), DELTA);
+ assertEquals((float) Timestamp.valueOf("2022-01-05 01:20:02").toEpochSecond(),
+ extractFloatFromLiteralValue("timestamp", "2022-01-05 01:20:02"), DELTA);
+ assertEquals((float) Timestamp.valueOf("2022-01-05 01:20:02").toEpochSecond(),
+ extractFloatFromLiteralValue("timestamp", "2022-1-5 01:20:02"), DELTA);
+ }
+
+ @Test
+ public void testLiteralExtractionFailures() {
+ // make sure the correct exceptions are raised so that we can default to standard computation
+ String[] types = {"int", "tinyint", "smallint", "bigint", "date", "timestamp", "float", "double"};
+ for (String type : types) {
+ // check we throw the correct exception when the boundary value parsing fails
+ assertThrows(IllegalArgumentException.class, () -> extractFloatFromLiteralValue(type, "abc"));
+ // check we throw the correct exception (NullPointerException for some, NumberFormatException for others,
+ // so we use their common parent RuntimeException) when a null value provided
+ assertThrows(RuntimeException.class, () -> extractFloatFromLiteralValue(type, null));
+ }
+
+ // check we throw the correct exception when an unsupported type is provided
+ assertThrows(IllegalStateException.class,
+ () -> extractFloatFromLiteralValue("typex", "abc"));
+ }
+
+ @Test
+ public void testBetweenLeftLowerThanMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDescLeq = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(4)));
+ long numRowsLeq = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDescLeq, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ ExprNodeDesc exprNodeDescBetween = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.FALSE),
+ COL_EXPR, createExprNodeConstantDesc(0), createExprNodeConstantDesc(4)));
+ long numRowsBetween = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDescBetween, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(numRowsLeq, numRowsBetween);
+ assertEquals(10, numRowsBetween);
+ }
+
+ @Test
+ public void testBetweenLeftLowerThanMinRightHigherThanMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.FALSE),
+ COL_EXPR, createExprNodeConstantDesc(0), createExprNodeConstantDesc(10)));
+ long numRowsBetween = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(VALUES.length, numRowsBetween);
+ }
+
+ @Test
+ public void testBetweenRightHigherThanMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDescGeq = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPEqualOrGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(2)));
+ long numRowsGeq = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDescGeq, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ ExprNodeDesc exprNodeDescBetween = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.FALSE),
+ COL_EXPR, createExprNodeConstantDesc(2), createExprNodeConstantDesc(10)));
+ long numRowsBetween = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDescBetween, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(numRowsGeq, numRowsBetween);
+ assertEquals(12, numRowsBetween);
+ }
+
+ @Test
+ public void testBetweenRightLowerThanMin() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDescBetween = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.FALSE),
+ COL_EXPR, createExprNodeConstantDesc(-1), createExprNodeConstantDesc(0)));
+ long numRowsBetween = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDescBetween, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRowsBetween);
+ }
+
+ @Test
+ public void testBetweenLeftHigherThanMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDescBetween = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.FALSE),
+ COL_EXPR, createExprNodeConstantDesc(10), createExprNodeConstantDesc(12)));
+ long numRowsBetween = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDescBetween, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRowsBetween);
+ }
+
+ @Test
+ public void testBetweenLeftEqualMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDescBetween = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.FALSE),
+ COL_EXPR, createExprNodeConstantDesc(3), createExprNodeConstantDesc(3)));
+ long numRowsBetween = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDescBetween, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(7, numRowsBetween);
+ }
+
+ @Test
+ public void testNotBetween() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.TRUE),
+ COL_EXPR, createExprNodeConstantDesc(3), createExprNodeConstantDesc(4)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ ExprNodeDesc exprNodeDescLth = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPLessThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(3)));
+ long numRowsLth = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDescLth, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ ExprNodeDesc exprNodeDescGth = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFOPGreaterThan(), Arrays.asList(COL_EXPR, createExprNodeConstantDesc(4)));
+ long numRowsGth = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDescGth, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(11, numRows);
+ assertEquals(numRows, numRowsLth + numRowsGth);
+ }
+
+ @Test
+ public void testNotBetweenLowerThanMinHigherThanMax() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.TRUE),
+ COL_EXPR, createExprNodeConstantDesc(0), createExprNodeConstantDesc(10)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(0, numRows);
+ }
+
+ @Test
+ public void testNotBetweenLeftEqualsRight() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.TRUE),
+ COL_EXPR, createExprNodeConstantDesc(3), createExprNodeConstantDesc(3)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(VALUES.length - 1, numRows);
+ }
+
+ @Test
+ public void testNotBetweenRightLowerThanLeft() throws SemanticException {
+ long numNulls = 2;
+ Statistics stats = createStatistics(VALUES, numNulls);
+
+ ExprNodeDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
+ new GenericUDFBetween(), Arrays.asList(new ExprNodeConstantDesc(Boolean.TRUE),
+ COL_EXPR, createExprNodeConstantDesc(4), createExprNodeConstantDesc(3)));
+ long numRows = new StatsRulesProcFactory.FilterStatsRule().evaluateExpression(
+ stats, exprNodeDesc, STATS_PROC_CTX, Collections.emptyList(), null, VALUES.length + numNulls);
+
+ assertEquals(VALUES.length, numRows);
+ }
+
+ private ExprNodeDesc createExprNodeConstantDesc(int value) {
+ return new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, value);
+ }
+
+ private Statistics createStatistics(long[] values, long numNulls) {
+ long numDVs = Arrays.stream(values).distinct().count();
+ Statistics stats = new Statistics(values.length + numNulls, 100, 100, 100);
+
+ HyperLogLog hll = StatisticsTestUtils.createHll(values);
+ float[] val = new float[values.length];
+ for (int i = 0; i < values.length; i++) {
+ val[i] = values[i];
+ }
+ KllFloatsSketch kll = StatisticsTestUtils.createKll(val);
+ ColStatistics colStatistics = createColStatistics(COL_NAME, "int", numNulls, numDVs, hll, kll);
+
+ stats.addToColumnStats(Collections.singletonList(colStatistics));
+
+ return stats;
+ }
+
+ private static ColStatistics createColStatistics(
+ String colName, String colType, long numNulls, long numDVs, Object hll, KllFloatsSketch kll) {
+ ColStatistics colStatistics = new ColStatistics(colName, colType);
+
+ colStatistics.setNumNulls(numNulls);
+ colStatistics.setCountDistint(numDVs);
+ if (hll != null) {
+ if (hll instanceof HyperLogLog) {
+ colStatistics.setBitVectors(((HyperLogLog) hll).serialize());
+ } else if (hll instanceof FMSketch) {
+ colStatistics.setBitVectors(((FMSketch) hll).serialize());
+ } else {
+ throw new IllegalArgumentException("Unsupported HLL class: " + hll.getClass().getName());
+ }
+ }
+ if (kll != null) {
+ colStatistics.setHistogram(kll.toByteArray());
+ }
+
+ return colStatistics;
+ }
+}
diff --git a/ql/src/test/queries/clientpositive/stats_histogram.q b/ql/src/test/queries/clientpositive/stats_histogram.q
new file mode 100644
index 00000000000..b98eaa349a4
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/stats_histogram.q
@@ -0,0 +1,57 @@
+set hive.stats.kll.enable=true;
+set metastore.stats.fetch.bitvector=true;
+set metastore.stats.fetch.kll=true;
+set hive.stats.autogather=true;
+set hive.stats.column.autogather=true;
+
+CREATE TABLE test_stats (a string, b int, c double, d float, e decimal(5,2), f timestamp, g date)
+STORED AS ORC;
+
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("a", 2, 1.1, 12.2, 1.3, "2020-11-2 00:00:00", "2020-11-2");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("b", 2, 2.1, NULL, 6.3, "2020-11-2 00:00:00", "2020-11-2");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("c", 2, 2.1, NULL, -8.3, "2020-11-2 00:00:00", "2020-11-02");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("d", 2, 3.1, 13.2, 10.2, "2020-11-2 00:00:00", "2020-11-2");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("e", 2, 3.1, 14.2, 10.2, "2020-11-02 00:00:00", "2020-11-2");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("f", 2, 4.1, NULL, 12.2, "2020-11-2 00:00:00", "2020-11-2");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("g", 2, 5.1, 15.2, -10.2, "2020-11-2 00:00:00", "2020-11-2");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("h", 2, 6.1, 16.2, 12.2, "2020-11-2 00:00:00", "2020-11-2");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("i", 3, 6.1, 17.2, 7.2, "2020-11-03 00:00:00", "2020-11-3");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("j", 4, NULL, 20.2, 1.2, "2020-11-4 00:00:00", "2020-11-4");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("k", 5, NULL, 50.2, -123.2, "2020-11-5 00:00:00", "2020-11-05");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("l", 6, NULL, 55.2, 1.2, "2020-11-6 00:00:00", "2020-11-6");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("m", 7, 9.1, 57.2, 1001.2, "2020-11-7 00:00:00", "2020-11-7");
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("n", NULL, 100.1, 1000.2, 0.2, NULL, NULL);
+INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("o", NULL, 101.1, 2000.2, -1.2, NULL, NULL);
+
+DESCRIBE FORMATTED test_stats;
+DESCRIBE FORMATTED test_stats a;
+DESCRIBE FORMATTED test_stats b;
+DESCRIBE FORMATTED test_stats c;
+DESCRIBE FORMATTED test_stats d;
+DESCRIBE FORMATTED test_stats e;
+DESCRIBE FORMATTED test_stats f;
+DESCRIBE FORMATTED test_stats g;
+
+EXPLAIN SELECT COUNT(*)
+FROM test_stats t1 JOIN test_stats t2 ON (t1.a = t2.a)
+WHERE t1.b BETWEEN 3 AND 5 AND t2.c > 6.0;
+SELECT COUNT(*)
+FROM test_stats t1 JOIN test_stats t2 ON (t1.a = t2.a)
+WHERE t1.b BETWEEN 3 AND 5 AND t2.c > 6.0;
+
+EXPLAIN SELECT COUNT(*) FROM test_stats WHERE b < 3;
+SELECT COUNT(*) FROM test_stats WHERE b < 3;
+
+EXPLAIN SELECT COUNT(*) FROM test_stats WHERE b >= 7;
+SELECT COUNT(*) FROM test_stats WHERE b >= 7;
+
+EXPLAIN SELECT COUNT(*) FROM test_stats WHERE d NOT BETWEEN 3 AND 7 AND e > 0;
+SELECT COUNT(*) FROM test_stats WHERE d NOT BETWEEN 3 AND 7 AND e > 0;
+
+EXPLAIN SELECT COUNT(*) FROM test_stats WHERE f >= "2020-11-7" AND g >= "2020-11-7";
+SELECT COUNT(*) FROM test_stats WHERE f >= "2020-11-7" AND g >= "2020-11-7";
+
+EXPLAIN SELECT COUNT(*) FROM test_stats WHERE f BETWEEN "2020-11-01" AND "2020-11-06" AND g >= "2020-11-01";
+SELECT COUNT(*) FROM test_stats WHERE f BETWEEN "2020-11-01" AND "2020-11-06" AND g >= "2020-11-01";
+
+DROP TABLE test_stats;
diff --git a/ql/src/test/results/clientpositive/llap/explain_ddl_nested_part.q.out b/ql/src/test/results/clientpositive/llap/explain_ddl_nested_part.q.out
index 93506105ce5..bba587992b9 100644
--- a/ql/src/test/results/clientpositive/llap/explain_ddl_nested_part.q.out
+++ b/ql/src/test/results/clientpositive/llap/explain_ddl_nested_part.q.out
@@ -96,15 +96,15 @@ ALTER TABLE default.test PARTITION (i=5,j=5) UPDATE STATISTICS SET('numRows'='1'
-- ALTER TABLE default.test ADD IF NOT EXISTS PARTITION (i=__HIVE_DEFAULT_PARTITION__,j=__HIVE_DEFAULT_PARTITION__);
-- ALTER TABLE default.test PARTITION (i=__HIVE_DEFAULT_PARTITION__,j=__HIVE_DEFAULT_PARTITION__) UPDATE STATISTICS SET('numRows'='2','rawDataSize'='4' );
ALTER TABLE default.test PARTITION (i=1,j=1) UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='1','highValue'='1','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test PARTITION i=1,j=1 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwfO+SA==
+-- BIT VECTORS PRESENT FOR default.test PARTITION i=1,j=1 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwfO+SA==
ALTER TABLE default.test PARTITION (i=2,j=2) UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='2','highValue'='2','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test PARTITION i=2,j=2 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBxIHguQQ=
+-- BIT VECTORS PRESENT FOR default.test PARTITION i=2,j=2 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBxIHguQQ=
ALTER TABLE default.test PARTITION (i=5,j=5) UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='5','highValue'='5','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test PARTITION i=5,j=5 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwaPu+AE=
+-- BIT VECTORS PRESENT FOR default.test PARTITION i=5,j=5 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwaPu+AE=
ALTER TABLE default.test PARTITION (i=3,j=3) UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='3','highValue'='3','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test PARTITION i=3,j=3 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgq/rqgE=
+-- BIT VECTORS PRESENT FOR default.test PARTITION i=3,j=3 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgq/rqgE=
ALTER TABLE default.test PARTITION (i=4,j=4) UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='4','highValue'='4','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test PARTITION i=4,j=4 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBhJHj9wE=
+-- BIT VECTORS PRESENT FOR default.test PARTITION i=4,j=4 FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBhJHj9wE=
-- ALTER TABLE default.test PARTITION (i=__HIVE_DEFAULT_PARTITION__,j=__HIVE_DEFAULT_PARTITION__) UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='0','highValue'='0','numNulls'='2','numDVs'='0' );
@@ -453,83 +453,83 @@ ALTER TABLE default.test2 PARTITION (i=9805326,j=9805326,s='9805326',t='9805326'
-- ALTER TABLE default.test2 ADD IF NOT EXISTS PARTITION (i=__HIVE_DEFAULT_PARTITION__,j=__HIVE_DEFAULT_PARTITION__,s='__HIVE_DEFAULT_PARTITION__',t='__HIVE_DEFAULT_PARTITION__');
-- ALTER TABLE default.test2 PARTITION (i=__HIVE_DEFAULT_PARTITION__,j=__HIVE_DEFAULT_PARTITION__,s='__HIVE_DEFAULT_PARTITION__',t='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS SET('numRows'='1','rawDataSize'='2' );
ALTER TABLE default.test2 PARTITION (i=1956691,j=1956691,s='1956691',t='1956691') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='1956691','highValue'='1956691','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1956691,j=1956691,s='1956691',t='1956691' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgZ3cVQ==
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1956691,j=1956691,s='1956691',t='1956691' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgZ3cVQ==
ALTER TABLE default.test2 PARTITION (i=7005576,j=7005576,s='7005576',t='7005576') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='7005576','highValue'='7005576','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=7005576,j=7005576,s='7005576',t='7005576' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgc7krgc=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=7005576,j=7005576,s='7005576',t='7005576' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgc7krgc=
ALTER TABLE default.test2 PARTITION (i=7286712,j=7286712,s='7286712',t='7286712') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='7286712','highValue'='7286712','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=7286712,j=7286712,s='7286712',t='7286712' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwperKQ==
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=7286712,j=7286712,s='7286712',t='7286712' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwperKQ==
ALTER TABLE default.test2 PARTITION (i=1522484,j=1522484,s='1522484',t='1522484') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='1522484','highValue'='1522484','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1522484,j=1522484,s='1522484',t='1522484' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBg5zSoAU=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1522484,j=1522484,s='1522484',t='1522484' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBg5zSoAU=
ALTER TABLE default.test2 PARTITION (i=2049769,j=2049769,s='2049769',t='2049769') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='2049769','highValue'='2049769','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=2049769,j=2049769,s='2049769',t='2049769' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwfS2sAQ=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=2049769,j=2049769,s='2049769',t='2049769' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwfS2sAQ=
ALTER TABLE default.test2 PARTITION (i=3073641,j=3073641,s='3073641',t='3073641') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='3073641','highValue'='3073641','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3073641,j=3073641,s='3073641',t='3073641' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBguu6jgQ=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3073641,j=3073641,s='3073641',t='3073641' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBguu6jgQ=
ALTER TABLE default.test2 PARTITION (i=3583871,j=3583871,s='3583871',t='3583871') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='3583871','highValue'='3583871','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3583871,j=3583871,s='3583871',t='3583871' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBxoKDzwM=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3583871,j=3583871,s='3583871',t='3583871' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBxoKDzwM=
ALTER TABLE default.test2 PARTITION (i=3847717,j=3847717,s='3847717',t='3847717') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='3847717','highValue'='3847717','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3847717,j=3847717,s='3847717',t='3847717' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBxfvAiwE=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3847717,j=3847717,s='3847717',t='3847717' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBxfvAiwE=
ALTER TABLE default.test2 PARTITION (i=7670676,j=7670676,s='7670676',t='7670676') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='7670676','highValue'='7670676','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=7670676,j=7670676,s='7670676',t='7670676' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgejUtQU=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=7670676,j=7670676,s='7670676',t='7670676' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgejUtQU=
ALTER TABLE default.test2 PARTITION (i=1655195,j=1655195,s='1655195',t='1655195') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='1655195','highValue'='1655195','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1655195,j=1655195,s='1655195',t='1655195' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwvLYBQ==
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1655195,j=1655195,s='1655195',t='1655195' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwvLYBQ==
ALTER TABLE default.test2 PARTITION (i=6129618,j=6129618,s='6129618',t='6129618') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='6129618','highValue'='6129618','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=6129618,j=6129618,s='6129618',t='6129618' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgqzpngY=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=6129618,j=6129618,s='6129618',t='6129618' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgqzpngY=
ALTER TABLE default.test2 PARTITION (i=7880229,j=7880229,s='7880229',t='7880229') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='7880229','highValue'='7880229','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=7880229,j=7880229,s='7880229',t='7880229' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgturhwI=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=7880229,j=7880229,s='7880229',t='7880229' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgturhwI=
ALTER TABLE default.test2 PARTITION (i=5803231,j=5803231,s='5803231',t='5803231') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='5803231','highValue'='5803231','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=5803231,j=5803231,s='5803231',t='5803231' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgpW7mQI=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=5803231,j=5803231,s='5803231',t='5803231' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgpW7mQI=
-- ALTER TABLE default.test2 PARTITION (i=__HIVE_DEFAULT_PARTITION__,j=__HIVE_DEFAULT_PARTITION__,s='3',t='3') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='0','highValue'='0','numNulls'='1','numDVs'='0' );
ALTER TABLE default.test2 PARTITION (i=4439186,j=4439186,s='4439186',t='4439186') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='4439186','highValue'='4439186','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4439186,j=4439186,s='4439186',t='4439186' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgd22gQY=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4439186,j=4439186,s='4439186',t='4439186' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgd22gQY=
ALTER TABLE default.test2 PARTITION (i=4497880,j=4497880,s='4497880',t='4497880') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='4497880','highValue'='4497880','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4497880,j=4497880,s='4497880',t='4497880' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwcvXgAE=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4497880,j=4497880,s='4497880',t='4497880' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwcvXgAE=
ALTER TABLE default.test2 PARTITION (i=2146641,j=2146641,s='2146641',t='2146641') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='2146641','highValue'='2146641','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=2146641,j=2146641,s='2146641',t='2146641' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBg9/xOw==
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=2146641,j=2146641,s='2146641',t='2146641' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBg9/xOw==
ALTER TABLE default.test2 PARTITION (i=967815,j=967815,s='967815',t='967815') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='967815','highValue'='967815','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=967815,j=967815,s='967815',t='967815' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgom62gM=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=967815,j=967815,s='967815',t='967815' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgom62gM=
ALTER TABLE default.test2 PARTITION (i=4604522,j=4604522,s='4604522',t='4604522') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='4604522','highValue'='4604522','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4604522,j=4604522,s='4604522',t='4604522' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgrrK+AY=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4604522,j=4604522,s='4604522',t='4604522' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgrrK+AY=
ALTER TABLE default.test2 PARTITION (i=5238051,j=5238051,s='5238051',t='5238051') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='5238051','highValue'='5238051','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=5238051,j=5238051,s='5238051',t='5238051' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBhfjCjwE=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=5238051,j=5238051,s='5238051',t='5238051' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBhfjCjwE=
ALTER TABLE default.test2 PARTITION (i=3187683,j=3187683,s='3187683',t='3187683') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='3187683','highValue'='3187683','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3187683,j=3187683,s='3187683',t='3187683' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwZn8uAc=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3187683,j=3187683,s='3187683',t='3187683' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwZn8uAc=
ALTER TABLE default.test2 PARTITION (i=6758530,j=6758530,s='6758530',t='6758530') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='6758530','highValue'='6758530','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=6758530,j=6758530,s='6758530',t='6758530' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgtSr0gQ=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=6758530,j=6758530,s='6758530',t='6758530' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgtSr0gQ=
ALTER TABLE default.test2 PARTITION (i=2581458,j=2581458,s='2581458',t='2581458') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='2581458','highValue'='2581458','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=2581458,j=2581458,s='2581458',t='2581458' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgf6ljQQ=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=2581458,j=2581458,s='2581458',t='2581458' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgf6ljQQ=
ALTER TABLE default.test2 PARTITION (i=4304398,j=4304398,s='4304398',t='4304398') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='4304398','highValue'='4304398','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4304398,j=4304398,s='4304398',t='4304398' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgf2tnwQ=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4304398,j=4304398,s='4304398',t='4304398' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgf2tnwQ=
ALTER TABLE default.test2 PARTITION (i=812837,j=812837,s='812837',t='812837') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='812837','highValue'='812837','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=812837,j=812837,s='812837',t='812837' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwuja+gY=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=812837,j=812837,s='812837',t='812837' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwuja+gY=
-- ALTER TABLE default.test2 PARTITION (i=2,j=2,s='__HIVE_DEFAULT_PARTITION__',t='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='2','highValue'='2','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=2,j=2,s='__HIVE_DEFAULT_PARTITION__',t='__HIVE_DEFAULT_PARTITION__' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBxIHguQQ=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=2,j=2,s='__HIVE_DEFAULT_PARTITION__',t='__HIVE_DEFAULT_PARTITION__' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBxIHguQQ=
ALTER TABLE default.test2 PARTITION (i=3356420,j=3356420,s='3356420',t='3356420') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='3356420','highValue'='3356420','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3356420,j=3356420,s='3356420',t='3356420' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwa/logE=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3356420,j=3356420,s='3356420',t='3356420' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwa/logE=
-- ALTER TABLE default.test2 PARTITION (i=__HIVE_DEFAULT_PARTITION__,j=__HIVE_DEFAULT_PARTITION__,s='__HIVE_DEFAULT_PARTITION__',t='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='0','highValue'='0','numNulls'='1','numDVs'='0' );
ALTER TABLE default.test2 PARTITION (i=42629,j=42629,s='42629',t='42629') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='42629','highValue'='42629','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=42629,j=42629,s='42629',t='42629' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBweiP4AY=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=42629,j=42629,s='42629',t='42629' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBweiP4AY=
ALTER TABLE default.test2 PARTITION (i=5247332,j=5247332,s='5247332',t='5247332') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='5247332','highValue'='5247332','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=5247332,j=5247332,s='5247332',t='5247332' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBg5mT4Qc=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=5247332,j=5247332,s='5247332',t='5247332' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBg5mT4Qc=
ALTER TABLE default.test2 PARTITION (i=1565998,j=1565998,s='1565998',t='1565998') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='1565998','highValue'='1565998','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1565998,j=1565998,s='1565998',t='1565998' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBhKGjdQ==
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1565998,j=1565998,s='1565998',t='1565998' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBhKGjdQ==
ALTER TABLE default.test2 PARTITION (i=1880938,j=1880938,s='1880938',t='1880938') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='1880938','highValue'='1880938','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1880938,j=1880938,s='1880938',t='1880938' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgb2svAQ=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1880938,j=1880938,s='1880938',t='1880938' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgb2svAQ=
ALTER TABLE default.test2 PARTITION (i=9805326,j=9805326,s='9805326',t='9805326') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='9805326','highValue'='9805326','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=9805326,j=9805326,s='9805326',t='9805326' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBw96RggU=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=9805326,j=9805326,s='9805326',t='9805326' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBw96RggU=
ALTER TABLE default.test2 PARTITION (i=3076080,j=3076080,s='3076080',t='3076080') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='3076080','highValue'='3076080','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3076080,j=3076080,s='3076080',t='3076080' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBweDt4QM=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3076080,j=3076080,s='3076080',t='3076080' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBweDt4QM=
ALTER TABLE default.test2 PARTITION (i=4409637,j=4409637,s='4409637',t='4409637') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='4409637','highValue'='4409637','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4409637,j=4409637,s='4409637',t='4409637' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgsT4sAc=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4409637,j=4409637,s='4409637',t='4409637' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgsT4sAc=
ALTER TABLE default.test2 PARTITION (i=6276003,j=6276003,s='6276003',t='6276003') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='6276003','highValue'='6276003','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=6276003,j=6276003,s='6276003',t='6276003' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBw8Kw8wQ=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=6276003,j=6276003,s='6276003',t='6276003' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBw8Kw8wQ=
-- ALTER TABLE default.test2 PARTITION (i=1,j=1,s='__HIVE_DEFAULT_PARTITION__',t='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='1','highValue'='1','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1,j=1,s='__HIVE_DEFAULT_PARTITION__',t='__HIVE_DEFAULT_PARTITION__' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwfO+SA==
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=1,j=1,s='__HIVE_DEFAULT_PARTITION__',t='__HIVE_DEFAULT_PARTITION__' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBwfO+SA==
ALTER TABLE default.test2 PARTITION (i=3023419,j=3023419,s='3023419',t='3023419') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='3023419','highValue'='3023419','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3023419,j=3023419,s='3023419',t='3023419' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBge2+4gQ=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=3023419,j=3023419,s='3023419',t='3023419' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBge2+4gQ=
ALTER TABLE default.test2 PARTITION (i=4115101,j=4115101,s='4115101',t='4115101') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='4115101','highValue'='4115101','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4115101,j=4115101,s='4115101',t='4115101' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgaamtAY=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=4115101,j=4115101,s='4115101',t='4115101' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBgaamtAY=
ALTER TABLE default.test2 PARTITION (i=971079,j=971079,s='971079',t='971079') UPDATE STATISTICS FOR COLUMN foo SET('lowValue'='971079','highValue'='971079','numNulls'='0','numDVs'='1' );
--- BIT VECTORS PRESENT FOR default.test2 PARTITION i=971079,j=971079,s='971079',t='971079' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBg6XR2gc=
+-- BIT VECTORS PRESENT FOR default.test2 PARTITION i=971079,j=971079,s='971079',t='971079' FOR COLUMN foo BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAEBg6XR2gc=
diff --git a/ql/src/test/results/clientpositive/llap/partition_explain_ddl.q.out b/ql/src/test/results/clientpositive/llap/partition_explain_ddl.q.out
index 83e97eb7d70..fe21a9af6f2 100644
--- a/ql/src/test/results/clientpositive/llap/partition_explain_ddl.q.out
+++ b/ql/src/test/results/clientpositive/llap/partition_explain_ddl.q.out
@@ -146,9 +146,9 @@ ALTER TABLE default.add_part_test UPDATE STATISTICS SET('numRows'='0','rawDataSi
ALTER TABLE default.add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-01');
ALTER TABLE default.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS SET('numRows'='19','rawDataSize'='267' );
ALTER TABLE default.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN key SET('avgColLen'='5.526315789473684','maxColLen'='6','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
+-- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
ALTER TABLE default.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN value SET('avgColLen'='7.526315789473684','maxColLen'='8','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
+-- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
@@ -271,13 +271,13 @@ ALTER TABLE default.add_part_test PARTITION (ds='2010-01-02') UPDATE STATISTICS
ALTER TABLE default.add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-03');
ALTER TABLE default.add_part_test PARTITION (ds='2010-01-03') UPDATE STATISTICS SET('numRows'='15','rawDataSize'='225' );
ALTER TABLE default.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN value SET('avgColLen'='7.526315789473684','maxColLen'='8','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
+-- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
ALTER TABLE default.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN key SET('avgColLen'='5.526315789473684','maxColLen'='6','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
+-- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
ALTER TABLE default.add_part_test PARTITION (ds='2010-01-03') UPDATE STATISTICS FOR COLUMN value SET('avgColLen'='8.0','maxColLen'='8','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-03' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgaOnjASAnPDu//////8BwfPgSP+i5ViAxKHO/P////8BgIjfnQTA44A8gJ2G1///////AcSq3Y7//////wH84vzz/f////8BhYzE6vz/////AfyQ+MsH/4aiEoChh/v//////wHBvOTP//////8B
+-- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-03' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgaOnjASAnPDu//////8BwfPgSP+i5ViAxKHO/P////8BgIjfnQTA44A8gJ2G1///////AcSq3Y7//////wH84vzz/f////8BhYzE6vz/////AfyQ+MsH/4aiEoChh/v//////wHBvOTP//////8B
ALTER TABLE default.add_part_test PARTITION (ds='2010-01-03') UPDATE STATISTICS FOR COLUMN key SET('avgColLen'='6.0','maxColLen'='6','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-03' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PxLrgqgHA37++Bv2T79H6/////wGAkZX+/v////8BwYzG8AWApP7O+f////8BgdfK4wT/m6bn+/////8B/9jt3wKC8sjC/f////8BvuyTzgPB1Mbz/P////8Bv5KKwgTBi8Ph+f////8BgIaQ5QM=
+-- BIT VECTORS PRESENT FOR default.add_part_test PARTITION ds='2010-01-03' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PxLrgqgHA37++Bv2T79H6/////wGAkZX+/v////8BwYzG8AWApP7O+f////8BgdfK4wT/m6bn+/////8B/9jt3wKC8sjC/f////8BvuyTzgPB1Mbz/P////8Bv5KKwgTBi8Ph+f////8BgIaQ5QM=
@@ -418,9 +418,9 @@ ALTER TABLE add_part_test_db.add_part_test UPDATE STATISTICS SET('numRows'='0','
ALTER TABLE add_part_test_db.add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-01');
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS SET('numRows'='19','rawDataSize'='267' );
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN key SET('avgColLen'='5.526315789473684','maxColLen'='6','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN value SET('avgColLen'='7.526315789473684','maxColLen'='8','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
@@ -543,13 +543,13 @@ ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-02') UPDATE ST
ALTER TABLE add_part_test_db.add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-03');
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-03') UPDATE STATISTICS SET('numRows'='15','rawDataSize'='225' );
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN value SET('avgColLen'='7.526315789473684','maxColLen'='8','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN key SET('avgColLen'='5.526315789473684','maxColLen'='6','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-03') UPDATE STATISTICS FOR COLUMN value SET('avgColLen'='8.0','maxColLen'='8','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-03' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgaOnjASAnPDu//////8BwfPgSP+i5ViAxKHO/P////8BgIjfnQTA44A8gJ2G1///////AcSq3Y7//////wH84vzz/f////8BhYzE6vz/////AfyQ+MsH/4aiEoChh/v//////wHBvOTP//////8B
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-03' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgaOnjASAnPDu//////8BwfPgSP+i5ViAxKHO/P////8BgIjfnQTA44A8gJ2G1///////AcSq3Y7//////wH84vzz/f////8BhYzE6vz/////AfyQ+MsH/4aiEoChh/v//////wHBvOTP//////8B
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-03') UPDATE STATISTICS FOR COLUMN key SET('avgColLen'='6.0','maxColLen'='6','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-03' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PxLrgqgHA37++Bv2T79H6/////wGAkZX+/v////8BwYzG8AWApP7O+f////8BgdfK4wT/m6bn+/////8B/9jt3wKC8sjC/f////8BvuyTzgPB1Mbz/P////8Bv5KKwgTBi8Ph+f////8BgIaQ5QM=
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-03' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PxLrgqgHA37++Bv2T79H6/////wGAkZX+/v////8BwYzG8AWApP7O+f////8BgdfK4wT/m6bn+/////8B/9jt3wKC8sjC/f////8BvuyTzgPB1Mbz/P////8Bv5KKwgTBi8Ph+f////8BgIaQ5QM=
@@ -645,13 +645,13 @@ ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-02') UPDATE ST
ALTER TABLE add_part_test_db.add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-03');
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-03') UPDATE STATISTICS SET('numRows'='15','rawDataSize'='225' );
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN value SET('avgColLen'='7.526315789473684','maxColLen'='8','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTwq6o+gLE0trKAvuXwdz6/////wGAp4ikAcCYgKYEgKv42f3/////AcLt7oMD/orRm///////AcDeotcBwOvPh/r/////AYDyz7kGwM+kvvz/////AYD90bn+/////wGBytzd/v////8Bv9fU1v//////AYDRh8UEwf6S/v3/////Ab+aqtIChMGNvP7/////AQ==
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-01') UPDATE STATISTICS FOR COLUMN key SET('avgColLen'='5.526315789473684','maxColLen'='6','numNulls'='0','numDVs'='19' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-01' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBMTga///AODuuFlwpGjgAL7r+CB+/////8BgPGY2AGB8bqnA8TDyvT6/////wG75PK0BMDg56b6/////wGAn/E7wM3HswOA1rnX/P////8BguujpwOB4qzYAv6j2hD/wYu4+v////8BgIKhwAGAge7KA8Cop9b//////wE=
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-03') UPDATE STATISTICS FOR COLUMN value SET('avgColLen'='8.0','maxColLen'='8','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-03' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgaOnjASAnPDu//////8BwfPgSP+i5ViAxKHO/P////8BgIjfnQTA44A8gJ2G1///////AcSq3Y7//////wH84vzz/f////8BhYzE6vz/////AfyQ+MsH/4aiEoChh/v//////wHBvOTP//////8B
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-03' FOR COLUMN value BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgaOnjASAnPDu//////8BwfPgSP+i5ViAxKHO/P////8BgIjfnQTA44A8gJ2G1///////AcSq3Y7//////wH84vzz/f////8BhYzE6vz/////AfyQ+MsH/4aiEoChh/v//////wHBvOTP//////8B
ALTER TABLE add_part_test_db.add_part_test PARTITION (ds='2010-01-03') UPDATE STATISTICS FOR COLUMN key SET('avgColLen'='6.0','maxColLen'='6','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-03' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PxLrgqgHA37++Bv2T79H6/////wGAkZX+/v////8BwYzG8AWApP7O+f////8BgdfK4wT/m6bn+/////8B/9jt3wKC8sjC/f////8BvuyTzgPB1Mbz/P////8Bv5KKwgTBi8Ph+f////8BgIaQ5QM=
+-- BIT VECTORS PRESENT FOR add_part_test_db.add_part_test PARTITION ds='2010-01-03' FOR COLUMN key BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PxLrgqgHA37++Bv2T79H6/////wGAkZX+/v////8BwYzG8AWApP7O+f////8BgdfK4wT/m6bn+/////8B/9jt3wKC8sjC/f////8BvuyTzgPB1Mbz/P////8Bv5KKwgTBi8Ph+f////8BgIaQ5QM=
@@ -899,65 +899,65 @@ ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Tennis') UPDATE
-- ALTER TABLE db_bdpbase.default_partition_test ADD IF NOT EXISTS PARTITION (sports='__HIVE_DEFAULT_PARTITION__');
-- ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS SET('numRows'='6','rawDataSize'='247' );
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Tennis') UPDATE STATISTICS FOR COLUMN lastname SET('avgColLen'='6.625','maxColLen'='10','numNulls'='0','numDVs'='16' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQw9jy4wbApZ6a/v////8B/sPU3fv/////AcCHp9kGgN/fv/r/////AYLFxPn+/////wGBkrK3Av2Ry8cCgc676Pv/////Ab/A89YBxOTiEvzixO/+/////wHD55ygA77R9+T6/////wHFq+mVB7vOpY39/////wE=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQw9jy4wbApZ6a/v////8B/sPU3fv/////AcCHp9kGgN/fv/r/////AYLFxPn+/////wGBkrK3Av2Ry8cCgc676Pv/////Ab/A89YBxOTiEvzixO/+/////wHD55ygA77R9+T6/////wHFq+mVB7vOpY39/////wE=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Tennis') UPDATE STATISTICS FOR COLUMN id SET('lowValue'='1007','highValue'='1086','numNulls'='0','numDVs'='16' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQgsirL7+NuZgFgJSyrf7/////AcDw1/3//////wGAw9/iAoGurff//////wH//tG+/v////8BhMDytf//////Abya/vb+/////wHCpbPM//////8B/v+D1v7/////AcD2p9r//////wGApJdwwLKYgQTByo7U//////8BwdS26vv/////AQ==
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQgsirL7+NuZgFgJSyrf7/////AcDw1/3//////wGAw9/iAoGurff//////wH//tG+/v////8BhMDytf//////Abya/vb+/////wHCpbPM//////8B/v+D1v7/////AcD2p9r//////wGApJdwwLKYgQTByo7U//////8BwdS26vv/////AQ==
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Tennis') UPDATE STATISTICS FOR COLUMN firstname SET('avgColLen'='5.875','maxColLen'='10','numNulls'='0','numDVs'='16' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQgp+Lswf/6/Gw+/////8BgM2YkAPB0+LF/f////8B/8ulC8DngZ0CgJ+o9v//////AcDR9AWBwp7N+v////8BwZT4/QH/qIfj//////8BgP+t//3/////Ab+a5YgDxoiV3QK8+K/t/v////8Bgdruk/7/////AQ==
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQgp+Lswf/6/Gw+/////8BgM2YkAPB0+LF/f////8B/8ulC8DngZ0CgJ+o9v//////AcDR9AWBwp7N+v////8BwZT4/QH/qIfj//////8BgP+t//3/////Ab+a5YgDxoiV3QK8+K/t/v////8Bgdruk/7/////AQ==
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Tennis') UPDATE STATISTICS FOR COLUMN country SET('avgColLen'='9.5625','maxColLen'='27','numNulls'='0','numDVs'='16' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQweClxwTAto6P/P////8Bg5qv8gO9nZOuAYWnjJT8/////wH7y9SkAoD13gGBtJOV/v////8BgqrYtf7/////Af3z25EEgbjb3wK/2Jud//////8BgZbmgvr/////Af+XtN4Dga/pvP//////AcH6mY8B
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQweClxwTAto6P/P////8Bg5qv8gO9nZOuAYWnjJT8/////wH7y9SkAoD13gGBtJOV/v////8BgqrYtf7/////Af3z25EEgbjb3wK/2Jud//////8BgZbmgvr/////Af+XtN4Dga/pvP//////AcH6mY8B
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Tennis') UPDATE STATISTICS FOR COLUMN city SET('avgColLen'='9.5625','maxColLen'='18','numNulls'='0','numDVs'='16' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQgfSH0gPAg7CJAoCl9ljC0YLK//////8B/picz/7/////AcDlwtP7/////wHAl84twIbP+v//////AYCoomaApeWJBoG1lab6/////wH/sNr5AoDjqJr8/////wHBz+68BcCH7fb9/////wH/+7xK
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Tennis' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBAQgfSH0gPAg7CJAoCl9ljC0YLK//////8B/picz/7/////AcDlwtP7/////wHAl84twIbP+v//////AYCoomaApeWJBoG1lab6/////wH/sNr5AoDjqJr8/////wHBz+68BcCH7fb9/////wH/+7xK
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Basketball') UPDATE STATISTICS FOR COLUMN lastname SET('avgColLen'='6.2','maxColLen'='8','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgZ/tvgSA18jgAoGZwtb+/////wH/zaeIAcDWpIP//////wGC/43R+/////8B/5/ogwGDhYerAfz89K4DgL6l8v7/////AYDwwZj//////wHAlpaV/f////8BgLqSK8GJg9D9/////wHCiZiWAw==
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgZ/tvgSA18jgAoGZwtb+/////wH/zaeIAcDWpIP//////wGC/43R+/////8B/5/ogwGDhYerAfz89K4DgL6l8v7/////AYDwwZj//////wHAlpaV/f////8BgLqSK8GJg9D9/////wHCiZiWAw==
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Basketball') UPDATE STATISTICS FOR COLUMN id SET('lowValue'='1001','highValue'='1097','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgYCNxQOA2pic/f////8BxuPc2AP8m7G4A7+Ix+z7/////wG/xNKo/f////8BgLCJswSAxK+U/f////8BgPmo2gHE6pNr/pSPjQK+4MTY+v////8BgIiW4ASEgZT0+/////8B/OCGvQE=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgYCNxQOA2pic/f////8BxuPc2AP8m7G4A7+Ix+z7/////wG/xNKo/f////8BgLCJswSAxK+U/f////8BgPmo2gHE6pNr/pSPjQK+4MTY+v////8BgIiW4ASEgZT0+/////8B/OCGvQE=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Basketball') UPDATE STATISTICS FOR COLUMN firstname SET('avgColLen'='6.466666666666667','maxColLen'='9','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgbilpAeC7+ssvtnsxfj/////AcCW6PAFwLLw9f7/////AcHwo4v9/////wH/nYz3AoGH1J/9/////wH/7Jv7/f////8BxNrM2wK85uvN/v////8BgJnvHMCdv9b//////wGAw7/oBcLAloD5/////wE=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgbilpAeC7+ssvtnsxfj/////AcCW6PAFwLLw9f7/////AcHwo4v9/////wH/nYz3AoGH1J/9/////wH/7Jv7/f////8BxNrM2wK85uvN/v////8BgJnvHMCdv9b//////wGAw7/oBcLAloD5/////wE=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Basketball') UPDATE STATISTICS FOR COLUMN country SET('avgColLen'='8.666666666666666','maxColLen'='16','numNulls'='0','numDVs'='14' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4OhdzCtAL8jYW5A8Ce5MX6/////wHAicoqgNKYqAGA+cYigKy/jwGB3qTH/f////8BgNG/hwLDkchH/YOnjf//////Af/knsj9/////wHA+9/oBICtjo7//////wE=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4OhdzCtAL8jYW5A8Ce5MX6/////wHAicoqgNKYqAGA+cYigKy/jwGB3qTH/f////8BgNG/hwLDkchH/YOnjf//////Af/knsj9/////wHA+9/oBICtjo7//////wE=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Basketball') UPDATE STATISTICS FOR COLUMN city SET('avgColLen'='9.733333333333333','maxColLen'='26','numNulls'='0','numDVs'='15' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgZjVoAbAiv9ng+yLbv3bzYP8/////wGFitRK/MrIowP/zYSG+/////8BwvzCl/7/////Af+jp8UBv+6Ctf//////AYGf/9wDv6/ZgQLCvb65+f////8BvsGr/QKAtqUo
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Basketball' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA8PgZjVoAbAiv9ng+yLbv3bzYP8/////wGFitRK/MrIowP/zYSG+/////8BwvzCl/7/////Af+jp8UBv+6Ctf//////AYGf/9wDv6/ZgQLCvb65+f////8BvsGr/QKAtqUo
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Football') UPDATE STATISTICS FOR COLUMN lastname SET('avgColLen'='5.6521739130434785','maxColLen'='8','numNulls'='0','numDVs'='23' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBcXgqW6lwS/7cK2AsGpjOz7/////wGCm5fW//////8BvbT96wTA5rbf/P////8Bgoq+hwO/6qXp/P////8B/8GzqP7/////AcCqq+L//////wGA5PWu/v////8Bwa37xwa/u+SQ/P////8Bwcrsrf3/////Ab+2k+ADgNahiAHAyNff/v////8BwNftnP7/////AYDDkyfB/tKCA4DswM8Bgoeop/3/////Af3g8/oB
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBcXgqW6lwS/7cK2AsGpjOz7/////wGCm5fW//////8BvbT96wTA5rbf/P////8Bgoq+hwO/6qXp/P////8B/8GzqP7/////AcCqq+L//////wGA5PWu/v////8Bwa37xwa/u+SQ/P////8Bwcrsrf3/////Ab+2k+ADgNahiAHAyNff/v////8BwNftnP7/////AYDDkyfB/tKCA4DswM8Bgoeop/3/////Af3g8/oB
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Football') UPDATE STATISTICS FOR COLUMN id SET('lowValue'='1006','highValue'='1100','numNulls'='0','numDVs'='23' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBcXgaCXhwXA1P+S/f////8Bg93J8wG91quf/P////8Bg8OW9gG9v/KA//////8Bhbns+AW7mZtJwM2xGcCm2JD7/////wHAzv2WAsDMyob9/////wHCpeL4/v////8Bv4iY4wW/56Sb+f////8BgLbX1QHB2tTlBP+Sqp36/////wGA9fLJAcGhkpcBgIXC0QPBxYbz//////8B/uGZ+vv/////AQ==
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBcXgaCXhwXA1P+S/f////8Bg93J8wG91quf/P////8Bg8OW9gG9v/KA//////8Bhbns+AW7mZtJwM2xGcCm2JD7/////wHAzv2WAsDMyob9/////wHCpeL4/v////8Bv4iY4wW/56Sb+f////8BgLbX1QHB2tTlBP+Sqp36/////wGA9fLJAcGhkpcBgIXC0QPBxYbz//////8B/uGZ+vv/////AQ==
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Football') UPDATE STATISTICS FOR COLUMN firstname SET('avgColLen'='5.782608695652174','maxColLen'='9','numNulls'='0','numDVs'='23' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBcXwbS80wPA08+OBMDEmZT6/////wHA7fi1BMLU84kBvs2zyfr/////AcDRiGfCgsLrA4DDgsb9/////wG+ipqK/v////8BwMjJowWC5Jyw//////8Bv5j19Pn/////AYDBosL//////wHA9t7fAb/QhPkDgJCxk/v/////AYKK0Qb+ra2ZAoOA5Pj//////wH9vuKJAcH15av8/////wGA1v/hAg==
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBcXwbS80wPA08+OBMDEmZT6/////wHA7fi1BMLU84kBvs2zyfr/////AcDRiGfCgsLrA4DDgsb9/////wG+ipqK/v////8BwMjJowWC5Jyw//////8Bv5j19Pn/////AYDBosL//////wHA9t7fAb/QhPkDgJCxk/v/////AYKK0Qb+ra2ZAoOA5Pj//////wH9vuKJAcH15av8/////wGA1v/hAg==
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Football') UPDATE STATISTICS FOR COLUMN country SET('avgColLen'='10.826086956521738','maxColLen'='33','numNulls'='0','numDVs'='22' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBYWw66T/gX/gadGwYr4n/3/////AYG57zXAkP/9+/////8BvbWmigfB/KHo+v////8BwYzq0gS/zsX3/f////8B//i05Pz/////AcG9m/4D//vz0fz/////AcCqpx3AxKqpAcCF+PACgIGGwPv/////AYSQlM0B/fqH6v//////Af+1/M38/////wGCp8si/pO8vQKC4MSEAQ==
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBYWw66T/gX/gadGwYr4n/3/////AYG57zXAkP/9+/////8BvbWmigfB/KHo+v////8BwYzq0gS/zsX3/f////8B//i05Pz/////AcG9m/4D//vz0fz/////AcCqpx3AxKqpAcCF+PACgIGGwPv/////AYSQlM0B/fqH6v//////Af+1/M38/////wGCp8si/pO8vQKC4MSEAQ==
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Football') UPDATE STATISTICS FOR COLUMN city SET('avgColLen'='8.782608695652174','maxColLen'='20','numNulls'='0','numDVs'='23' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBcXw7/xvgW/vaexAoD6oqj6/////wHBqdfw//////8BvuPJ1P7/////AYCN2LgDwaXjsv3/////Af+IxJcGwbyujvv/////Af+ttFnBqYmkA7+Yrqb7/////wHAgr+LBIGHpJP9/////wH/iY3oAYDAj7/8/////wGAie6wAYD20A3Cw5oi/ozl0QKAl+LVAcHRpASBoNS9//////8B
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Football' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBcXw7/xvgW/vaexAoD6oqj6/////wHBqdfw//////8BvuPJ1P7/////AYCN2LgDwaXjsv3/////Af+IxJcGwbyujvv/////Af+ttFnBqYmkA7+Yrqb7/////wHAgr+LBIGHpJP9/////wH/iY3oAYDAj7/8/////wGAie6wAYD20A3Cw5oi/ozl0QKAl+LVAcHRpASBoNS9//////8B
-- ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS FOR COLUMN lastname SET('avgColLen'='6.333333333333333','maxColLen'='8','numNulls'='0','numDVs'='6' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGgbePuQfB9sjg+P////8Bw93qogW83dm9/P////8BwYbtqQK/yYTa/v////8B
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGgbePuQfB9sjg+P////8Bw93qogW83dm9/P////8BwYbtqQK/yYTa/v////8B
-- ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS FOR COLUMN id SET('lowValue'='1005','highValue'='1098','numNulls'='0','numDVs'='6' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGgeXMkgKAqeDgAYDOgWfAjve+/P////8Bga6jv///////Af/B69cD
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGgeXMkgKAqeDgAYDOgWfAjve+/P////8Bga6jv///////Af/B69cD
-- ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS FOR COLUMN firstname SET('avgColLen'='7.0','maxColLen'='9','numNulls'='0','numDVs'='6' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGwe3GFsGq05QBv6XU8P7/////AYHCq/AGwJ7v2Pr/////AcG15yw=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGwe3GFsGq05QBv6XU8P7/////AYHCq/AGwJ7v2Pr/////AcG15yw=
-- ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS FOR COLUMN country SET('avgColLen'='9.333333333333334','maxColLen'='15','numNulls'='0','numDVs'='6' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGwaXO9gaA4928+f////8BgoXl0AP/u9nv/v////8BwP3t9gP/wbzN/P////8B
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGwaXO9gaA4928+f////8BgoXl0AP/u9nv/v////8BwP3t9gP/wbzN/P////8B
-- ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='__HIVE_DEFAULT_PARTITION__') UPDATE STATISTICS FOR COLUMN city SET('avgColLen'='10.5','maxColLen'='17','numNulls'='0','numDVs'='6' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGwaGlyQKAyt+3AYCs6oQBwt6jv/7/////Af/P9r4C//fig/v/////AQ==
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='__HIVE_DEFAULT_PARTITION__' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoAYGwaGlyQKAyt+3AYCs6oQBwt6jv/7/////Af/P9r4C//fig/v/////AQ==
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Cricket') UPDATE STATISTICS FOR COLUMN lastname SET('avgColLen'='6.642857142857143','maxColLen'='10','numNulls'='0','numDVs'='14' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4Oh/eB3gH6kIqEBoDm49H7/////wGBzfvyAcDCzlTC2eaU/P////8B/q3H8v3/////Af+bxdoFgJL6jPv/////AYG84W//1I1igqC6pwH+kKX+/v////8Bwr362gQ=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4Oh/eB3gH6kIqEBoDm49H7/////wGBzfvyAcDCzlTC2eaU/P////8B/q3H8v3/////Af+bxdoFgJL6jPv/////AYG84W//1I1igqC6pwH+kKX+/v////8Bwr362gQ=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Cricket') UPDATE STATISTICS FOR COLUMN id SET('lowValue'='1002','highValue'='1099','numNulls'='0','numDVs'='14' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4Owp+5vgXAgIGC+/////8B/6frowTArIixAcCvrknA3Jmv//////8BgIHmBoGA38n6/////wHAhO5lgs76yAG+9dT6Ar+cjO36/////wGAx7GYBML+3Mj8/////wE=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4Owp+5vgXAgIGC+/////8B/6frowTArIixAcCvrknA3Jmv//////8BgIHmBoGA38n6/////wHAhO5lgs76yAG+9dT6Ar+cjO36/////wGAx7GYBML+3Mj8/////wE=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Cricket') UPDATE STATISTICS FOR COLUMN firstname SET('avgColLen'='5.285714285714286','maxColLen'='9','numNulls'='0','numDVs'='14' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4OwbS80wOB6fyi/f////8B/4eU6QKB+P2a/f////8Bw7z4qQT8+8KdAsKaipb9/////wH/pLfZ//////8Bg/367/3/////Af72zOj//////wG+8a95gvbxuf3/////Ab7LzcwFwNTv8Pz/////AQ==
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4OwbS80wOB6fyi/f////8B/4eU6QKB+P2a/f////8Bw7z4qQT8+8KdAsKaipb9/////wH/pLfZ//////8Bg/367/3/////Af72zOj//////wG+8a95gvbxuf3/////Ab7LzcwFwNTv8Pz/////AQ==
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Cricket') UPDATE STATISTICS FOR COLUMN country SET('avgColLen'='9.714285714285714','maxColLen'='18','numNulls'='0','numDVs'='14' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4OwanPogfEsvOR+/////8B/M6b8AKCibP3/f////8Bvprl2gLB+fLA//////8B/7zUlv7/////AcXwxLz+/////wG7+JuG/v////8BgPWXlAHB1KGNAv/45vkDwvKSh/n/////AYSysbYC
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4OwanPogfEsvOR+/////8B/M6b8AKCibP3/f////8Bvprl2gLB+fLA//////8B/7zUlv7/////AcXwxLz+/////wG7+JuG/v////8BgPWXlAHB1KGNAv/45vkDwvKSh/n/////AYSysbYC
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Cricket') UPDATE STATISTICS FOR COLUMN city SET('avgColLen'='9.5','maxColLen'='18','numNulls'='0','numDVs'='14' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4Owv2KsweAxJ3Y+f////8Bv7aO5QaAo7XI+v////8BwM6p9/3/////AcCGkuoBwYfi6f//////Ab/3+6ICxPu9zwP90YX2//////8BgpXXv/j/////Af3c9qcEgZnkk/z/////Af/BmTE=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Cricket' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoA4Owv2KsweAxJ3Y+f////8Bv7aO5QaAo7XI+v////8BwM6p9/3/////AcCGkuoBwYfi6f//////Ab/3+6ICxPu9zwP90YX2//////8BgpXXv/j/////Af3c9qcEgZnkk/z/////Af/BmTE=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Badminton') UPDATE STATISTICS FOR COLUMN lastname SET('avgColLen'='6.269230769230769','maxColLen'='9','numNulls'='0','numDVs'='26' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBoaw5bxWsDKgKgE/pbs5vz/////AYDw18AEguubl/3/////AcHBj579/////wH9pu6ABYKuotL+/////wH+x+S0//////8BgMj+2QLAjorG+f////8BwJm1pwLBgbeMBL/BgaD8/////wHAuJDKA4CXh93//////wHBuPXl+v////8B//SC/wWAweCB//////8BwMTBhv7/////AYC+l/cBwI3g1fr/////AYTay4cBvtbOsAL//sj+A8Hd+ZH4/////wE=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN lastname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBoaw5bxWsDKgKgE/pbs5vz/////AYDw18AEguubl/3/////AcHBj579/////wH9pu6ABYKuotL+/////wH+x+S0//////8BgMj+2QLAjorG+f////8BwJm1pwLBgbeMBL/BgaD8/////wHAuJDKA4CXh93//////wHBuPXl+v////8B//SC/wWAweCB//////8BwMTBhv7/////AYC+l/cBwI3g1fr/////AYTay4cBvtbOsAL//sj+A8Hd+ZH4/////wE=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Badminton') UPDATE STATISTICS FOR COLUMN id SET('lowValue'='1004','highValue'='1094','numNulls'='0','numDVs'='26' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBoahd3Cdf32590G/6SLj/3/////AYCykT7Av92e/f////8BwYfGqwP/9oKWAYD4z638/////wGBwrxzv+i89wHA0Niv/f////8BgLL2rgGAstnL+/////8BguuQrAH+1vPaBMG99/D5/////wH//4P6BcDSr2WAnKXH/P////8BgKfB3P3/////AYCTgJEEgqb8+Pv/////AcCCvmq+0O7IAoDBhvb+/////wHA0dlC
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN id BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBoahd3Cdf32590G/6SLj/3/////AYCykT7Av92e/f////8BwYfGqwP/9oKWAYD4z638/////wGBwrxzv+i89wHA0Niv/f////8BgLL2rgGAstnL+/////8BguuQrAH+1vPaBMG99/D5/////wH//4P6BcDSr2WAnKXH/P////8BgKfB3P3/////AYCTgJEEgqb8+Pv/////AcCCvmq+0O7IAoDBhvb+/////wHA0dlC
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Badminton') UPDATE STATISTICS FOR COLUMN firstname SET('avgColLen'='5.615384615384615','maxColLen'='8','numNulls'='0','numDVs'='25' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBkZyJCAxAK5lvTQBMDlqKP5/////wHAl7CAA8CbmRzA8faZ/f////8Bgf/q/f//////Ab/LlfIGgNTwqvz/////AcDcuH6Ay4aWAoDC4QXA6L+q/f////8BwPv+XoDLzxPAzvF9waDs/Pz/////AcCQvbL+/////wHAuJKC//////8Bv8CUgAXAzNZPw+WL8vz/////AYD4gCH9tourAcPksWU=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN firstname BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBkZyJCAxAK5lvTQBMDlqKP5/////wHAl7CAA8CbmRzA8faZ/f////8Bgf/q/f//////Ab/LlfIGgNTwqvz/////AcDcuH6Ay4aWAoDC4QXA6L+q/f////8BwPv+XoDLzxPAzvF9waDs/Pz/////AcCQvbL+/////wHAuJKC//////8Bv8CUgAXAzNZPw+WL8vz/////AYD4gCH9tourAcPksWU=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Badminton') UPDATE STATISTICS FOR COLUMN country SET('avgColLen'='10.038461538461538','maxColLen'='44','numNulls'='0','numDVs'='26' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBoahbLBzAP85PKJ/f////8BwLe62gLD4vSXAYGN3q0B/6Xhi/r/////Ab2osqUFwqq+oQH+tdPa+v////8BgPenYoCwhkKAndeXA4Sn15/+/////wH+le6UAr/epnP/7bOp+v////8BgKHJV4DwkcACg4u68wGB85+S/f////8B/JyLowOAs7kWgYmk8fr/////Af/OrLoDgO721P//////AYGQ3sD7/////wE=
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN country BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBoahbLBzAP85PKJ/f////8BwLe62gLD4vSXAYGN3q0B/6Xhi/r/////Ab2osqUFwqq+oQH+tdPa+v////8BgPenYoCwhkKAndeXA4Sn15/+/////wH+le6UAr/epnP/7bOp+v////8BgKHJV4DwkcACg4u68wGB85+S/f////8B/JyLowOAs7kWgYmk8fr/////Af/OrLoDgO721P//////AYGQ3sD7/////wE=
ALTER TABLE db_bdpbase.default_partition_test PARTITION (sports='Badminton') UPDATE STATISTICS FOR COLUMN city SET('avgColLen'='7.961538461538462','maxColLen'='15','numNulls'='0','numDVs'='26' );
--- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBoagcnkoQHDmv3pA/6E55gChOm3qP//////AbzE6Yj7/////wH/jeLHAYC7qMgEgO/pvP//////AYDl7q35/////wHAnoCTBsDeqIT+/////wGArM7x//////8BwdCEL//10sL+/////wGB/Yxrv/G7V8HV1cT7/////wGA/62HA4C7sfsBwJT/rPr/////AYCFgrIBgbbW6P//////Ab707dv//////wGEkeKsBYLSvzy6mZuX+v////8B
+-- BIT VECTORS PRESENT FOR db_bdpbase.default_partition_test PARTITION sports='Badminton' FOR COLUMN city BUT THEY ARE NOT SUPPORTED YET. THE BASE64 VALUE FOR THE BITVECTOR IS SExMoBoagcnkoQHDmv3pA/6E55gChOm3qP//////AbzE6Yj7/////wH/jeLHAYC7qMgEgO/pvP//////AYDl7q35/////wHAnoCTBsDeqIT+/////wGArM7x//////8BwdCEL//10sL+/////wGB/Yxrv/G7V8HV1cT7/////wGA/62HA4C7sfsBwJT/rPr/////AYCFgrIBgbbW6P//////Ab707dv//////wGEkeKsBYLSvzy6mZuX+v////8B
diff --git a/ql/src/test/results/clientpositive/llap/stats_histogram.q.out b/ql/src/test/results/clientpositive/llap/stats_histogram.q.out
new file mode 100644
index 00000000000..ab433500e7b
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/stats_histogram.q.out
@@ -0,0 +1,921 @@
+PREHOOK: query: CREATE TABLE test_stats (a string, b int, c double, d float, e decimal(5,2), f timestamp, g date)
+STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: CREATE TABLE test_stats (a string, b int, c double, d float, e decimal(5,2), f timestamp, g date)
+STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_stats
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("a", 2, 1.1, 12.2, 1.3, "2020-11-2 00:00:00", "2020-11-2")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("a", 2, 1.1, 12.2, 1.3, "2020-11-2 00:00:00", "2020-11-2")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("b", 2, 2.1, NULL, 6.3, "2020-11-2 00:00:00", "2020-11-2")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("b", 2, 2.1, NULL, 6.3, "2020-11-2 00:00:00", "2020-11-2")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d EXPRESSION []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("c", 2, 2.1, NULL, -8.3, "2020-11-2 00:00:00", "2020-11-02")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("c", 2, 2.1, NULL, -8.3, "2020-11-2 00:00:00", "2020-11-02")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d EXPRESSION []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("d", 2, 3.1, 13.2, 10.2, "2020-11-2 00:00:00", "2020-11-2")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("d", 2, 3.1, 13.2, 10.2, "2020-11-2 00:00:00", "2020-11-2")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("e", 2, 3.1, 14.2, 10.2, "2020-11-02 00:00:00", "2020-11-2")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("e", 2, 3.1, 14.2, 10.2, "2020-11-02 00:00:00", "2020-11-2")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("f", 2, 4.1, NULL, 12.2, "2020-11-2 00:00:00", "2020-11-2")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("f", 2, 4.1, NULL, 12.2, "2020-11-2 00:00:00", "2020-11-2")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d EXPRESSION []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("g", 2, 5.1, 15.2, -10.2, "2020-11-2 00:00:00", "2020-11-2")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("g", 2, 5.1, 15.2, -10.2, "2020-11-2 00:00:00", "2020-11-2")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("h", 2, 6.1, 16.2, 12.2, "2020-11-2 00:00:00", "2020-11-2")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("h", 2, 6.1, 16.2, 12.2, "2020-11-2 00:00:00", "2020-11-2")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("i", 3, 6.1, 17.2, 7.2, "2020-11-03 00:00:00", "2020-11-3")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("i", 3, 6.1, 17.2, 7.2, "2020-11-03 00:00:00", "2020-11-3")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("j", 4, NULL, 20.2, 1.2, "2020-11-4 00:00:00", "2020-11-4")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("j", 4, NULL, 20.2, 1.2, "2020-11-4 00:00:00", "2020-11-4")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c EXPRESSION []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("k", 5, NULL, 50.2, -123.2, "2020-11-5 00:00:00", "2020-11-05")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("k", 5, NULL, 50.2, -123.2, "2020-11-5 00:00:00", "2020-11-05")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c EXPRESSION []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("l", 6, NULL, 55.2, 1.2, "2020-11-6 00:00:00", "2020-11-6")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("l", 6, NULL, 55.2, 1.2, "2020-11-6 00:00:00", "2020-11-6")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c EXPRESSION []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("m", 7, 9.1, 57.2, 1001.2, "2020-11-7 00:00:00", "2020-11-7")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("m", 7, 9.1, 57.2, 1001.2, "2020-11-7 00:00:00", "2020-11-7")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b SCRIPT []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f SCRIPT []
+POSTHOOK: Lineage: test_stats.g SCRIPT []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("n", NULL, 100.1, 1000.2, 0.2, NULL, NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("n", NULL, 100.1, 1000.2, 0.2, NULL, NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b EXPRESSION []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f EXPRESSION []
+POSTHOOK: Lineage: test_stats.g EXPRESSION []
+PREHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("o", NULL, 101.1, 2000.2, -1.2, NULL, NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: INSERT INTO test_stats (a, b, c, d, e, f, g) VALUES ("o", NULL, 101.1, 2000.2, -1.2, NULL, NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_stats
+POSTHOOK: Lineage: test_stats.a SCRIPT []
+POSTHOOK: Lineage: test_stats.b EXPRESSION []
+POSTHOOK: Lineage: test_stats.c SCRIPT []
+POSTHOOK: Lineage: test_stats.d SCRIPT []
+POSTHOOK: Lineage: test_stats.e SCRIPT []
+POSTHOOK: Lineage: test_stats.f EXPRESSION []
+POSTHOOK: Lineage: test_stats.g EXPRESSION []
+PREHOOK: query: DESCRIBE FORMATTED test_stats
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test_stats
+POSTHOOK: query: DESCRIBE FORMATTED test_stats
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test_stats
+# col_name data_type comment
+a string
+b int
+c double
+d float
+e decimal(5,2)
+f timestamp
+g date
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\",\"f\":\"true\",\"g\":\"true\"}}
+ bucketing_version 2
+ numFiles 15
+ numRows 15
+ rawDataSize 4287
+ totalSize 11029
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: DESCRIBE FORMATTED test_stats a
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test_stats
+POSTHOOK: query: DESCRIBE FORMATTED test_stats a
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test_stats
+col_name a
+data_type string
+min
+max
+num_nulls 0
+distinct_count 15
+avg_col_len 1.0
+max_col_len 1
+num_trues
+num_falses
+bit_vector HL
+histogram
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\",\"f\":\"true\",\"g\":\"true\"}}
+PREHOOK: query: DESCRIBE FORMATTED test_stats b
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test_stats
+POSTHOOK: query: DESCRIBE FORMATTED test_stats b
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test_stats
+col_name b
+data_type int
+min 2
+max 7
+num_nulls 2
+distinct_count 6
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector HL
+histogram Q1: 2, Q2: 2, Q3: 4
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\",\"f\":\"true\",\"g\":\"true\"}}
+PREHOOK: query: DESCRIBE FORMATTED test_stats c
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test_stats
+POSTHOOK: query: DESCRIBE FORMATTED test_stats c
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test_stats
+col_name c
+data_type double
+min 1.1
+max 101.1
+num_nulls 3
+distinct_count 9
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector HL
+histogram Q1: 3.1, Q2: 5.1, Q3: 9.1
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\",\"f\":\"true\",\"g\":\"true\"}}
+PREHOOK: query: DESCRIBE FORMATTED test_stats d
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test_stats
+POSTHOOK: query: DESCRIBE FORMATTED test_stats d
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test_stats
+col_name d
+data_type float
+min 12.199999809265137
+max 2000.199951171875
+num_nulls 3
+distinct_count 12
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector HL
+histogram Q1: 15.2, Q2: 20.2, Q3: 57.2
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\",\"f\":\"true\",\"g\":\"true\"}}
+PREHOOK: query: DESCRIBE FORMATTED test_stats e
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test_stats
+POSTHOOK: query: DESCRIBE FORMATTED test_stats e
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test_stats
+col_name e
+data_type decimal(5,2)
+min -10.2
+max 12.2
+num_nulls 1
+distinct_count 11
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector HL
+histogram Q1: -1.2, Q2: 1.3, Q3: 10.2
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\",\"f\":\"true\",\"g\":\"true\"}}
+PREHOOK: query: DESCRIBE FORMATTED test_stats f
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test_stats
+POSTHOOK: query: DESCRIBE FORMATTED test_stats f
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test_stats
+col_name f
+data_type timestamp
+min 2020-11-02 00:00:00
+max 2020-11-07 00:00:00
+num_nulls 2
+distinct_count 6
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector HL
+histogram Q1: 2020-11-02 00:00:00, Q2: 2020-11-02 00:00:00, Q3: 2020-11-04 00:00:00
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\",\"f\":\"true\",\"g\":\"true\"}}
+PREHOOK: query: DESCRIBE FORMATTED test_stats g
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test_stats
+POSTHOOK: query: DESCRIBE FORMATTED test_stats g
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test_stats
+col_name g
+data_type date
+min 2020-11-02
+max 2020-11-07
+num_nulls 2
+distinct_count 6
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector HL
+histogram Q1: 2020-11-02, Q2: 2020-11-02, Q3: 2020-11-04
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\",\"f\":\"true\",\"g\":\"true\"}}
+PREHOOK: query: EXPLAIN SELECT COUNT(*)
+FROM test_stats t1 JOIN test_stats t2 ON (t1.a = t2.a)
+WHERE t1.b BETWEEN 3 AND 5 AND t2.c > 6.0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT COUNT(*)
+FROM test_stats t1 JOIN test_stats t2 ON (t1.a = t2.a)
+WHERE t1.b BETWEEN 3 AND 5 AND t2.c > 6.0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ filterExpr: (b BETWEEN 3 AND 5 and a is not null) (type: boolean)
+ Statistics: Num rows: 15 Data size: 1331 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (b BETWEEN 3 AND 5 and a is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 267 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: a (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: t2
+ filterExpr: ((c > 6.0D) and a is not null) (type: boolean)
+ Statistics: Num rows: 15 Data size: 1379 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: ((c > 6.0D) and a is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: a (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 425 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 5 Data size: 425 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.6666666
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT COUNT(*)
+FROM test_stats t1 JOIN test_stats t2 ON (t1.a = t2.a)
+WHERE t1.b BETWEEN 3 AND 5 AND t2.c > 6.0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*)
+FROM test_stats t1 JOIN test_stats t2 ON (t1.a = t2.a)
+WHERE t1.b BETWEEN 3 AND 5 AND t2.c > 6.0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+1
+PREHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE b < 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE b < 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: test_stats
+ filterExpr: (b < 3) (type: boolean)
+ Statistics: Num rows: 15 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (b < 3) (type: boolean)
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.875
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT COUNT(*) FROM test_stats WHERE b < 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM test_stats WHERE b < 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+8
+PREHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE b >= 7
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE b >= 7
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: test_stats
+ filterExpr: (b >= 7) (type: boolean)
+ Statistics: Num rows: 15 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (b >= 7) (type: boolean)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT COUNT(*) FROM test_stats WHERE b >= 7
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM test_stats WHERE b >= 7
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+1
+PREHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE d NOT BETWEEN 3 AND 7 AND e > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE d NOT BETWEEN 3 AND 7 AND e > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: test_stats
+ filterExpr: (d NOT BETWEEN 3.0 AND 7.0 and (e > 0)) (type: boolean)
+ Statistics: Num rows: 15 Data size: 1732 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (d NOT BETWEEN 3.0 AND 7.0 and (e > 0)) (type: boolean)
+ Statistics: Num rows: 10 Data size: 1156 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 10 Data size: 1156 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.9
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT COUNT(*) FROM test_stats WHERE d NOT BETWEEN 3 AND 7 AND e > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM test_stats WHERE d NOT BETWEEN 3 AND 7 AND e > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+8
+PREHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE f >= "2020-11-7" AND g >= "2020-11-7"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE f >= "2020-11-7" AND g >= "2020-11-7"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: test_stats
+ filterExpr: ((g >= DATE'2020-11-07') and (f >= TIMESTAMP'2020-11-07 00:00:00')) (type: boolean)
+ Statistics: Num rows: 15 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: ((g >= DATE'2020-11-07') and (f >= TIMESTAMP'2020-11-07 00:00:00')) (type: boolean)
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT COUNT(*) FROM test_stats WHERE f >= "2020-11-7" AND g >= "2020-11-7"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM test_stats WHERE f >= "2020-11-7" AND g >= "2020-11-7"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+1
+PREHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE f BETWEEN "2020-11-01" AND "2020-11-06" AND g >= "2020-11-01"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT COUNT(*) FROM test_stats WHERE f BETWEEN "2020-11-01" AND "2020-11-06" AND g >= "2020-11-01"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: test_stats
+ filterExpr: ((g >= DATE'2020-11-01') and f BETWEEN TIMESTAMP'2020-11-01 00:00:00' AND TIMESTAMP'2020-11-06 00:00:00') (type: boolean)
+ Statistics: Num rows: 15 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: ((g >= DATE'2020-11-01') and f BETWEEN TIMESTAMP'2020-11-01 00:00:00' AND TIMESTAMP'2020-11-06 00:00:00') (type: boolean)
+ Statistics: Num rows: 11 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 11 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.9090909
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT COUNT(*) FROM test_stats WHERE f BETWEEN "2020-11-01" AND "2020-11-06" AND g >= "2020-11-01"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM test_stats WHERE f BETWEEN "2020-11-01" AND "2020-11-06" AND g >= "2020-11-01"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_stats
+#### A masked pattern was here ####
+12
+PREHOOK: query: DROP TABLE test_stats
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@test_stats
+PREHOOK: Output: default@test_stats
+POSTHOOK: query: DROP TABLE test_stats
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@test_stats
+POSTHOOK: Output: default@test_stats
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 9a043797ce3..9fe8dbb410c 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -8546,6 +8546,11 @@ void DoubleColumnStatsData::__set_bitVectors(const std::string& val) {
this->bitVectors = val;
__isset.bitVectors = true;
}
+
+void DoubleColumnStatsData::__set_histogram(const std::string& val) {
+ this->histogram = val;
+__isset.histogram = true;
+}
std::ostream& operator<<(std::ostream& out, const DoubleColumnStatsData& obj)
{
obj.printTo(out);
@@ -8616,6 +8621,14 @@ uint32_t DoubleColumnStatsData::read(::apache::thrift::protocol::TProtocol* ipro
xfer += iprot->skip(ftype);
}
break;
+ case 6:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readBinary(this->histogram);
+ this->__isset.histogram = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
default:
xfer += iprot->skip(ftype);
break;
@@ -8660,6 +8673,11 @@ uint32_t DoubleColumnStatsData::write(::apache::thrift::protocol::TProtocol* opr
xfer += oprot->writeBinary(this->bitVectors);
xfer += oprot->writeFieldEnd();
}
+ if (this->__isset.histogram) {
+ xfer += oprot->writeFieldBegin("histogram", ::apache::thrift::protocol::T_STRING, 6);
+ xfer += oprot->writeBinary(this->histogram);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -8672,6 +8690,7 @@ void swap(DoubleColumnStatsData &a, DoubleColumnStatsData &b) {
swap(a.numNulls, b.numNulls);
swap(a.numDVs, b.numDVs);
swap(a.bitVectors, b.bitVectors);
+ swap(a.histogram, b.histogram);
swap(a.__isset, b.__isset);
}
@@ -8681,6 +8700,7 @@ DoubleColumnStatsData::DoubleColumnStatsData(const DoubleColumnStatsData& other3
numNulls = other312.numNulls;
numDVs = other312.numDVs;
bitVectors = other312.bitVectors;
+ histogram = other312.histogram;
__isset = other312.__isset;
}
DoubleColumnStatsData& DoubleColumnStatsData::operator=(const DoubleColumnStatsData& other313) {
@@ -8689,6 +8709,7 @@ DoubleColumnStatsData& DoubleColumnStatsData::operator=(const DoubleColumnStatsD
numNulls = other313.numNulls;
numDVs = other313.numDVs;
bitVectors = other313.bitVectors;
+ histogram = other313.histogram;
__isset = other313.__isset;
return *this;
}
@@ -8700,6 +8721,7 @@ void DoubleColumnStatsData::printTo(std::ostream& out) const {
out << ", " << "numNulls=" << to_string(numNulls);
out << ", " << "numDVs=" << to_string(numDVs);
out << ", " << "bitVectors="; (__isset.bitVectors ? (out << to_string(bitVectors)) : (out << "<null>"));
+ out << ", " << "histogram="; (__isset.histogram ? (out << to_string(histogram)) : (out << "<null>"));
out << ")";
}
@@ -8730,6 +8752,11 @@ void LongColumnStatsData::__set_bitVectors(const std::string& val) {
this->bitVectors = val;
__isset.bitVectors = true;
}
+
+void LongColumnStatsData::__set_histogram(const std::string& val) {
+ this->histogram = val;
+__isset.histogram = true;
+}
std::ostream& operator<<(std::ostream& out, const LongColumnStatsData& obj)
{
obj.printTo(out);
@@ -8800,6 +8827,14 @@ uint32_t LongColumnStatsData::read(::apache::thrift::protocol::TProtocol* iprot)
xfer += iprot->skip(ftype);
}
break;
+ case 6:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readBinary(this->histogram);
+ this->__isset.histogram = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
default:
xfer += iprot->skip(ftype);
break;
@@ -8844,6 +8879,11 @@ uint32_t LongColumnStatsData::write(::apache::thrift::protocol::TProtocol* oprot
xfer += oprot->writeBinary(this->bitVectors);
xfer += oprot->writeFieldEnd();
}
+ if (this->__isset.histogram) {
+ xfer += oprot->writeFieldBegin("histogram", ::apache::thrift::protocol::T_STRING, 6);
+ xfer += oprot->writeBinary(this->histogram);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -8856,6 +8896,7 @@ void swap(LongColumnStatsData &a, LongColumnStatsData &b) {
swap(a.numNulls, b.numNulls);
swap(a.numDVs, b.numDVs);
swap(a.bitVectors, b.bitVectors);
+ swap(a.histogram, b.histogram);
swap(a.__isset, b.__isset);
}
@@ -8865,6 +8906,7 @@ LongColumnStatsData::LongColumnStatsData(const LongColumnStatsData& other314) {
numNulls = other314.numNulls;
numDVs = other314.numDVs;
bitVectors = other314.bitVectors;
+ histogram = other314.histogram;
__isset = other314.__isset;
}
LongColumnStatsData& LongColumnStatsData::operator=(const LongColumnStatsData& other315) {
@@ -8873,6 +8915,7 @@ LongColumnStatsData& LongColumnStatsData::operator=(const LongColumnStatsData& o
numNulls = other315.numNulls;
numDVs = other315.numDVs;
bitVectors = other315.bitVectors;
+ histogram = other315.histogram;
__isset = other315.__isset;
return *this;
}
@@ -8884,6 +8927,7 @@ void LongColumnStatsData::printTo(std::ostream& out) const {
out << ", " << "numNulls=" << to_string(numNulls);
out << ", " << "numDVs=" << to_string(numDVs);
out << ", " << "bitVectors="; (__isset.bitVectors ? (out << to_string(bitVectors)) : (out << "<null>"));
+ out << ", " << "histogram="; (__isset.histogram ? (out << to_string(histogram)) : (out << "<null>"));
out << ")";
}
@@ -9378,6 +9422,11 @@ void DecimalColumnStatsData::__set_bitVectors(const std::string& val) {
this->bitVectors = val;
__isset.bitVectors = true;
}
+
+void DecimalColumnStatsData::__set_histogram(const std::string& val) {
+ this->histogram = val;
+__isset.histogram = true;
+}
std::ostream& operator<<(std::ostream& out, const DecimalColumnStatsData& obj)
{
obj.printTo(out);
@@ -9448,6 +9497,14 @@ uint32_t DecimalColumnStatsData::read(::apache::thrift::protocol::TProtocol* ipr
xfer += iprot->skip(ftype);
}
break;
+ case 6:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readBinary(this->histogram);
+ this->__isset.histogram = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
default:
xfer += iprot->skip(ftype);
break;
@@ -9492,6 +9549,11 @@ uint32_t DecimalColumnStatsData::write(::apache::thrift::protocol::TProtocol* op
xfer += oprot->writeBinary(this->bitVectors);
xfer += oprot->writeFieldEnd();
}
+ if (this->__isset.histogram) {
+ xfer += oprot->writeFieldBegin("histogram", ::apache::thrift::protocol::T_STRING, 6);
+ xfer += oprot->writeBinary(this->histogram);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -9504,6 +9566,7 @@ void swap(DecimalColumnStatsData &a, DecimalColumnStatsData &b) {
swap(a.numNulls, b.numNulls);
swap(a.numDVs, b.numDVs);
swap(a.bitVectors, b.bitVectors);
+ swap(a.histogram, b.histogram);
swap(a.__isset, b.__isset);
}
@@ -9513,6 +9576,7 @@ DecimalColumnStatsData::DecimalColumnStatsData(const DecimalColumnStatsData& oth
numNulls = other322.numNulls;
numDVs = other322.numDVs;
bitVectors = other322.bitVectors;
+ histogram = other322.histogram;
__isset = other322.__isset;
}
DecimalColumnStatsData& DecimalColumnStatsData::operator=(const DecimalColumnStatsData& other323) {
@@ -9521,6 +9585,7 @@ DecimalColumnStatsData& DecimalColumnStatsData::operator=(const DecimalColumnSta
numNulls = other323.numNulls;
numDVs = other323.numDVs;
bitVectors = other323.bitVectors;
+ histogram = other323.histogram;
__isset = other323.__isset;
return *this;
}
@@ -9532,6 +9597,7 @@ void DecimalColumnStatsData::printTo(std::ostream& out) const {
out << ", " << "numNulls=" << to_string(numNulls);
out << ", " << "numDVs=" << to_string(numDVs);
out << ", " << "bitVectors="; (__isset.bitVectors ? (out << to_string(bitVectors)) : (out << "<null>"));
+ out << ", " << "histogram="; (__isset.histogram ? (out << to_string(histogram)) : (out << "<null>"));
out << ")";
}
@@ -9654,6 +9720,11 @@ void DateColumnStatsData::__set_bitVectors(const std::string& val) {
this->bitVectors = val;
__isset.bitVectors = true;
}
+
+void DateColumnStatsData::__set_histogram(const std::string& val) {
+ this->histogram = val;
+__isset.histogram = true;
+}
std::ostream& operator<<(std::ostream& out, const DateColumnStatsData& obj)
{
obj.printTo(out);
@@ -9724,6 +9795,14 @@ uint32_t DateColumnStatsData::read(::apache::thrift::protocol::TProtocol* iprot)
xfer += iprot->skip(ftype);
}
break;
+ case 6:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readBinary(this->histogram);
+ this->__isset.histogram = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
default:
xfer += iprot->skip(ftype);
break;
@@ -9768,6 +9847,11 @@ uint32_t DateColumnStatsData::write(::apache::thrift::protocol::TProtocol* oprot
xfer += oprot->writeBinary(this->bitVectors);
xfer += oprot->writeFieldEnd();
}
+ if (this->__isset.histogram) {
+ xfer += oprot->writeFieldBegin("histogram", ::apache::thrift::protocol::T_STRING, 6);
+ xfer += oprot->writeBinary(this->histogram);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -9780,6 +9864,7 @@ void swap(DateColumnStatsData &a, DateColumnStatsData &b) {
swap(a.numNulls, b.numNulls);
swap(a.numDVs, b.numDVs);
swap(a.bitVectors, b.bitVectors);
+ swap(a.histogram, b.histogram);
swap(a.__isset, b.__isset);
}
@@ -9789,6 +9874,7 @@ DateColumnStatsData::DateColumnStatsData(const DateColumnStatsData& other326) {
numNulls = other326.numNulls;
numDVs = other326.numDVs;
bitVectors = other326.bitVectors;
+ histogram = other326.histogram;
__isset = other326.__isset;
}
DateColumnStatsData& DateColumnStatsData::operator=(const DateColumnStatsData& other327) {
@@ -9797,6 +9883,7 @@ DateColumnStatsData& DateColumnStatsData::operator=(const DateColumnStatsData& o
numNulls = other327.numNulls;
numDVs = other327.numDVs;
bitVectors = other327.bitVectors;
+ histogram = other327.histogram;
__isset = other327.__isset;
return *this;
}
@@ -9808,6 +9895,7 @@ void DateColumnStatsData::printTo(std::ostream& out) const {
out << ", " << "numNulls=" << to_string(numNulls);
out << ", " << "numDVs=" << to_string(numDVs);
out << ", " << "bitVectors="; (__isset.bitVectors ? (out << to_string(bitVectors)) : (out << "<null>"));
+ out << ", " << "histogram="; (__isset.histogram ? (out << to_string(histogram)) : (out << "<null>"));
out << ")";
}
@@ -9930,6 +10018,11 @@ void TimestampColumnStatsData::__set_bitVectors(const std::string& val) {
this->bitVectors = val;
__isset.bitVectors = true;
}
+
+void TimestampColumnStatsData::__set_histogram(const std::string& val) {
+ this->histogram = val;
+__isset.histogram = true;
+}
std::ostream& operator<<(std::ostream& out, const TimestampColumnStatsData& obj)
{
obj.printTo(out);
@@ -10000,6 +10093,14 @@ uint32_t TimestampColumnStatsData::read(::apache::thrift::protocol::TProtocol* i
xfer += iprot->skip(ftype);
}
break;
+ case 6:
+ if (ftype == ::apache::thrift::protocol::T_STRING) {
+ xfer += iprot->readBinary(this->histogram);
+ this->__isset.histogram = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
default:
xfer += iprot->skip(ftype);
break;
@@ -10044,6 +10145,11 @@ uint32_t TimestampColumnStatsData::write(::apache::thrift::protocol::TProtocol*
xfer += oprot->writeBinary(this->bitVectors);
xfer += oprot->writeFieldEnd();
}
+ if (this->__isset.histogram) {
+ xfer += oprot->writeFieldBegin("histogram", ::apache::thrift::protocol::T_STRING, 6);
+ xfer += oprot->writeBinary(this->histogram);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -10056,6 +10162,7 @@ void swap(TimestampColumnStatsData &a, TimestampColumnStatsData &b) {
swap(a.numNulls, b.numNulls);
swap(a.numDVs, b.numDVs);
swap(a.bitVectors, b.bitVectors);
+ swap(a.histogram, b.histogram);
swap(a.__isset, b.__isset);
}
@@ -10065,6 +10172,7 @@ TimestampColumnStatsData::TimestampColumnStatsData(const TimestampColumnStatsDat
numNulls = other330.numNulls;
numDVs = other330.numDVs;
bitVectors = other330.bitVectors;
+ histogram = other330.histogram;
__isset = other330.__isset;
}
TimestampColumnStatsData& TimestampColumnStatsData::operator=(const TimestampColumnStatsData& other331) {
@@ -10073,6 +10181,7 @@ TimestampColumnStatsData& TimestampColumnStatsData::operator=(const TimestampCol
numNulls = other331.numNulls;
numDVs = other331.numDVs;
bitVectors = other331.bitVectors;
+ histogram = other331.histogram;
__isset = other331.__isset;
return *this;
}
@@ -10084,6 +10193,7 @@ void TimestampColumnStatsData::printTo(std::ostream& out) const {
out << ", " << "numNulls=" << to_string(numNulls);
out << ", " << "numDVs=" << to_string(numDVs);
out << ", " << "bitVectors="; (__isset.bitVectors ? (out << to_string(bitVectors)) : (out << "<null>"));
+ out << ", " << "histogram="; (__isset.histogram ? (out << to_string(histogram)) : (out << "<null>"));
out << ")";
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
index b57f953746e..9f48895c79c 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@ -3943,10 +3943,11 @@ void swap(BooleanColumnStatsData &a, BooleanColumnStatsData &b);
std::ostream& operator<<(std::ostream& out, const BooleanColumnStatsData& obj);
typedef struct _DoubleColumnStatsData__isset {
- _DoubleColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false) {}
+ _DoubleColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false), histogram(false) {}
bool lowValue :1;
bool highValue :1;
bool bitVectors :1;
+ bool histogram :1;
} _DoubleColumnStatsData__isset;
class DoubleColumnStatsData : public virtual ::apache::thrift::TBase {
@@ -3959,7 +3960,8 @@ class DoubleColumnStatsData : public virtual ::apache::thrift::TBase {
highValue(0),
numNulls(0),
numDVs(0),
- bitVectors() {
+ bitVectors(),
+ histogram() {
}
virtual ~DoubleColumnStatsData() noexcept;
@@ -3968,6 +3970,7 @@ class DoubleColumnStatsData : public virtual ::apache::thrift::TBase {
int64_t numNulls;
int64_t numDVs;
std::string bitVectors;
+ std::string histogram;
_DoubleColumnStatsData__isset __isset;
@@ -3981,6 +3984,8 @@ class DoubleColumnStatsData : public virtual ::apache::thrift::TBase {
void __set_bitVectors(const std::string& val);
+ void __set_histogram(const std::string& val);
+
bool operator == (const DoubleColumnStatsData & rhs) const
{
if (__isset.lowValue != rhs.__isset.lowValue)
@@ -3999,6 +4004,10 @@ class DoubleColumnStatsData : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
return false;
+ if (__isset.histogram != rhs.__isset.histogram)
+ return false;
+ else if (__isset.histogram && !(histogram == rhs.histogram))
+ return false;
return true;
}
bool operator != (const DoubleColumnStatsData &rhs) const {
@@ -4018,10 +4027,11 @@ void swap(DoubleColumnStatsData &a, DoubleColumnStatsData &b);
std::ostream& operator<<(std::ostream& out, const DoubleColumnStatsData& obj);
typedef struct _LongColumnStatsData__isset {
- _LongColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false) {}
+ _LongColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false), histogram(false) {}
bool lowValue :1;
bool highValue :1;
bool bitVectors :1;
+ bool histogram :1;
} _LongColumnStatsData__isset;
class LongColumnStatsData : public virtual ::apache::thrift::TBase {
@@ -4034,7 +4044,8 @@ class LongColumnStatsData : public virtual ::apache::thrift::TBase {
highValue(0),
numNulls(0),
numDVs(0),
- bitVectors() {
+ bitVectors(),
+ histogram() {
}
virtual ~LongColumnStatsData() noexcept;
@@ -4043,6 +4054,7 @@ class LongColumnStatsData : public virtual ::apache::thrift::TBase {
int64_t numNulls;
int64_t numDVs;
std::string bitVectors;
+ std::string histogram;
_LongColumnStatsData__isset __isset;
@@ -4056,6 +4068,8 @@ class LongColumnStatsData : public virtual ::apache::thrift::TBase {
void __set_bitVectors(const std::string& val);
+ void __set_histogram(const std::string& val);
+
bool operator == (const LongColumnStatsData & rhs) const
{
if (__isset.lowValue != rhs.__isset.lowValue)
@@ -4074,6 +4088,10 @@ class LongColumnStatsData : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
return false;
+ if (__isset.histogram != rhs.__isset.histogram)
+ return false;
+ else if (__isset.histogram && !(histogram == rhs.histogram))
+ return false;
return true;
}
bool operator != (const LongColumnStatsData &rhs) const {
@@ -4268,10 +4286,11 @@ void swap(Decimal &a, Decimal &b);
std::ostream& operator<<(std::ostream& out, const Decimal& obj);
typedef struct _DecimalColumnStatsData__isset {
- _DecimalColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false) {}
+ _DecimalColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false), histogram(false) {}
bool lowValue :1;
bool highValue :1;
bool bitVectors :1;
+ bool histogram :1;
} _DecimalColumnStatsData__isset;
class DecimalColumnStatsData : public virtual ::apache::thrift::TBase {
@@ -4282,7 +4301,8 @@ class DecimalColumnStatsData : public virtual ::apache::thrift::TBase {
DecimalColumnStatsData() noexcept
: numNulls(0),
numDVs(0),
- bitVectors() {
+ bitVectors(),
+ histogram() {
}
virtual ~DecimalColumnStatsData() noexcept;
@@ -4291,6 +4311,7 @@ class DecimalColumnStatsData : public virtual ::apache::thrift::TBase {
int64_t numNulls;
int64_t numDVs;
std::string bitVectors;
+ std::string histogram;
_DecimalColumnStatsData__isset __isset;
@@ -4304,6 +4325,8 @@ class DecimalColumnStatsData : public virtual ::apache::thrift::TBase {
void __set_bitVectors(const std::string& val);
+ void __set_histogram(const std::string& val);
+
bool operator == (const DecimalColumnStatsData & rhs) const
{
if (__isset.lowValue != rhs.__isset.lowValue)
@@ -4322,6 +4345,10 @@ class DecimalColumnStatsData : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
return false;
+ if (__isset.histogram != rhs.__isset.histogram)
+ return false;
+ else if (__isset.histogram && !(histogram == rhs.histogram))
+ return false;
return true;
}
bool operator != (const DecimalColumnStatsData &rhs) const {
@@ -4378,10 +4405,11 @@ void swap(Date &a, Date &b);
std::ostream& operator<<(std::ostream& out, const Date& obj);
typedef struct _DateColumnStatsData__isset {
- _DateColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false) {}
+ _DateColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false), histogram(false) {}
bool lowValue :1;
bool highValue :1;
bool bitVectors :1;
+ bool histogram :1;
} _DateColumnStatsData__isset;
class DateColumnStatsData : public virtual ::apache::thrift::TBase {
@@ -4392,7 +4420,8 @@ class DateColumnStatsData : public virtual ::apache::thrift::TBase {
DateColumnStatsData() noexcept
: numNulls(0),
numDVs(0),
- bitVectors() {
+ bitVectors(),
+ histogram() {
}
virtual ~DateColumnStatsData() noexcept;
@@ -4401,6 +4430,7 @@ class DateColumnStatsData : public virtual ::apache::thrift::TBase {
int64_t numNulls;
int64_t numDVs;
std::string bitVectors;
+ std::string histogram;
_DateColumnStatsData__isset __isset;
@@ -4414,6 +4444,8 @@ class DateColumnStatsData : public virtual ::apache::thrift::TBase {
void __set_bitVectors(const std::string& val);
+ void __set_histogram(const std::string& val);
+
bool operator == (const DateColumnStatsData & rhs) const
{
if (__isset.lowValue != rhs.__isset.lowValue)
@@ -4432,6 +4464,10 @@ class DateColumnStatsData : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
return false;
+ if (__isset.histogram != rhs.__isset.histogram)
+ return false;
+ else if (__isset.histogram && !(histogram == rhs.histogram))
+ return false;
return true;
}
bool operator != (const DateColumnStatsData &rhs) const {
@@ -4488,10 +4524,11 @@ void swap(Timestamp &a, Timestamp &b);
std::ostream& operator<<(std::ostream& out, const Timestamp& obj);
typedef struct _TimestampColumnStatsData__isset {
- _TimestampColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false) {}
+ _TimestampColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false), histogram(false) {}
bool lowValue :1;
bool highValue :1;
bool bitVectors :1;
+ bool histogram :1;
} _TimestampColumnStatsData__isset;
class TimestampColumnStatsData : public virtual ::apache::thrift::TBase {
@@ -4502,7 +4539,8 @@ class TimestampColumnStatsData : public virtual ::apache::thrift::TBase {
TimestampColumnStatsData() noexcept
: numNulls(0),
numDVs(0),
- bitVectors() {
+ bitVectors(),
+ histogram() {
}
virtual ~TimestampColumnStatsData() noexcept;
@@ -4511,6 +4549,7 @@ class TimestampColumnStatsData : public virtual ::apache::thrift::TBase {
int64_t numNulls;
int64_t numDVs;
std::string bitVectors;
+ std::string histogram;
_TimestampColumnStatsData__isset __isset;
@@ -4524,6 +4563,8 @@ class TimestampColumnStatsData : public virtual ::apache::thrift::TBase {
void __set_bitVectors(const std::string& val);
+ void __set_histogram(const std::string& val);
+
bool operator == (const TimestampColumnStatsData & rhs) const
{
if (__isset.lowValue != rhs.__isset.lowValue)
@@ -4542,6 +4583,10 @@ class TimestampColumnStatsData : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
return false;
+ if (__isset.histogram != rhs.__isset.histogram)
+ return false;
+ else if (__isset.histogram && !(histogram == rhs.histogram))
+ return false;
return true;
}
bool operator != (const TimestampColumnStatsData &rhs) const {
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DateColumnStatsData.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DateColumnStatsData.java
index 0a10401616a..6d439d90298 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DateColumnStatsData.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DateColumnStatsData.java
@@ -16,6 +16,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final org.apache.thrift.protocol.TField NUM_NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("numNulls", org.apache.thrift.protocol.TType.I64, (short)3);
private static final org.apache.thrift.protocol.TField NUM_DVS_FIELD_DESC = new org.apache.thrift.protocol.TField("numDVs", org.apache.thrift.protocol.TType.I64, (short)4);
private static final org.apache.thrift.protocol.TField BIT_VECTORS_FIELD_DESC = new org.apache.thrift.protocol.TField("bitVectors", org.apache.thrift.protocol.TType.STRING, (short)5);
+ private static final org.apache.thrift.protocol.TField HISTOGRAM_FIELD_DESC = new org.apache.thrift.protocol.TField("histogram", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new DateColumnStatsDataStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new DateColumnStatsDataTupleSchemeFactory();
@@ -25,6 +26,7 @@ package org.apache.hadoop.hive.metastore.api;
private long numNulls; // required
private long numDVs; // required
private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer bitVectors; // optional
+ private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -32,7 +34,8 @@ package org.apache.hadoop.hive.metastore.api;
HIGH_VALUE((short)2, "highValue"),
NUM_NULLS((short)3, "numNulls"),
NUM_DVS((short)4, "numDVs"),
- BIT_VECTORS((short)5, "bitVectors");
+ BIT_VECTORS((short)5, "bitVectors"),
+ HISTOGRAM((short)6, "histogram");
private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
@@ -58,6 +61,8 @@ package org.apache.hadoop.hive.metastore.api;
return NUM_DVS;
case 5: // BIT_VECTORS
return BIT_VECTORS;
+ case 6: // HISTOGRAM
+ return HISTOGRAM;
default:
return null;
}
@@ -102,7 +107,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final int __NUMNULLS_ISSET_ID = 0;
private static final int __NUMDVS_ISSET_ID = 1;
private byte __isset_bitfield = 0;
- private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS};
+ private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS,_Fields.HISTOGRAM};
public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -116,6 +121,8 @@ package org.apache.hadoop.hive.metastore.api;
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
tmpMap.put(_Fields.BIT_VECTORS, new org.apache.thrift.meta_data.FieldMetaData("bitVectors", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
+ tmpMap.put(_Fields.HISTOGRAM, new org.apache.thrift.meta_data.FieldMetaData("histogram", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(DateColumnStatsData.class, metaDataMap);
}
@@ -150,6 +157,9 @@ package org.apache.hadoop.hive.metastore.api;
if (other.isSetBitVectors()) {
this.bitVectors = org.apache.thrift.TBaseHelper.copyBinary(other.bitVectors);
}
+ if (other.isSetHistogram()) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(other.histogram);
+ }
}
public DateColumnStatsData deepCopy() {
@@ -165,6 +175,7 @@ package org.apache.hadoop.hive.metastore.api;
setNumDVsIsSet(false);
this.numDVs = 0;
this.bitVectors = null;
+ this.histogram = null;
}
@org.apache.thrift.annotation.Nullable
@@ -291,6 +302,38 @@ package org.apache.hadoop.hive.metastore.api;
}
}
+ public byte[] getHistogram() {
+ setHistogram(org.apache.thrift.TBaseHelper.rightSize(histogram));
+ return histogram == null ? null : histogram.array();
+ }
+
+ public java.nio.ByteBuffer bufferForHistogram() {
+ return org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void setHistogram(byte[] histogram) {
+ this.histogram = histogram == null ? (java.nio.ByteBuffer)null : java.nio.ByteBuffer.wrap(histogram.clone());
+ }
+
+ public void setHistogram(@org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void unsetHistogram() {
+ this.histogram = null;
+ }
+
+ /** Returns true if field histogram is set (has been assigned a value) and false otherwise */
+ public boolean isSetHistogram() {
+ return this.histogram != null;
+ }
+
+ public void setHistogramIsSet(boolean value) {
+ if (!value) {
+ this.histogram = null;
+ }
+ }
+
public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
switch (field) {
case LOW_VALUE:
@@ -337,6 +380,18 @@ package org.apache.hadoop.hive.metastore.api;
}
break;
+ case HISTOGRAM:
+ if (value == null) {
+ unsetHistogram();
+ } else {
+ if (value instanceof byte[]) {
+ setHistogram((byte[])value);
+ } else {
+ setHistogram((java.nio.ByteBuffer)value);
+ }
+ }
+ break;
+
}
}
@@ -358,6 +413,9 @@ package org.apache.hadoop.hive.metastore.api;
case BIT_VECTORS:
return getBitVectors();
+ case HISTOGRAM:
+ return getHistogram();
+
}
throw new java.lang.IllegalStateException();
}
@@ -379,6 +437,8 @@ package org.apache.hadoop.hive.metastore.api;
return isSetNumDVs();
case BIT_VECTORS:
return isSetBitVectors();
+ case HISTOGRAM:
+ return isSetHistogram();
}
throw new java.lang.IllegalStateException();
}
@@ -441,6 +501,15 @@ package org.apache.hadoop.hive.metastore.api;
return false;
}
+ boolean this_present_histogram = true && this.isSetHistogram();
+ boolean that_present_histogram = true && that.isSetHistogram();
+ if (this_present_histogram || that_present_histogram) {
+ if (!(this_present_histogram && that_present_histogram))
+ return false;
+ if (!this.histogram.equals(that.histogram))
+ return false;
+ }
+
return true;
}
@@ -464,6 +533,10 @@ package org.apache.hadoop.hive.metastore.api;
if (isSetBitVectors())
hashCode = hashCode * 8191 + bitVectors.hashCode();
+ hashCode = hashCode * 8191 + ((isSetHistogram()) ? 131071 : 524287);
+ if (isSetHistogram())
+ hashCode = hashCode * 8191 + histogram.hashCode();
+
return hashCode;
}
@@ -525,6 +598,16 @@ package org.apache.hadoop.hive.metastore.api;
return lastComparison;
}
}
+ lastComparison = java.lang.Boolean.compare(isSetHistogram(), other.isSetHistogram());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetHistogram()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.histogram, other.histogram);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -583,6 +666,16 @@ package org.apache.hadoop.hive.metastore.api;
}
first = false;
}
+ if (isSetHistogram()) {
+ if (!first) sb.append(", ");
+ sb.append("histogram:");
+ if (this.histogram == null) {
+ sb.append("null");
+ } else {
+ org.apache.thrift.TBaseHelper.toString(this.histogram, sb);
+ }
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -684,6 +777,14 @@ package org.apache.hadoop.hive.metastore.api;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 6: // HISTOGRAM
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -724,6 +825,13 @@ package org.apache.hadoop.hive.metastore.api;
oprot.writeFieldEnd();
}
}
+ if (struct.histogram != null) {
+ if (struct.isSetHistogram()) {
+ oprot.writeFieldBegin(HISTOGRAM_FIELD_DESC);
+ oprot.writeBinary(struct.histogram);
+ oprot.writeFieldEnd();
+ }
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -753,7 +861,10 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
optionals.set(2);
}
- oprot.writeBitSet(optionals, 3);
+ if (struct.isSetHistogram()) {
+ optionals.set(3);
+ }
+ oprot.writeBitSet(optionals, 4);
if (struct.isSetLowValue()) {
struct.lowValue.write(oprot);
}
@@ -763,6 +874,9 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
oprot.writeBinary(struct.bitVectors);
}
+ if (struct.isSetHistogram()) {
+ oprot.writeBinary(struct.histogram);
+ }
}
@Override
@@ -772,7 +886,7 @@ package org.apache.hadoop.hive.metastore.api;
struct.setNumNullsIsSet(true);
struct.numDVs = iprot.readI64();
struct.setNumDVsIsSet(true);
- java.util.BitSet incoming = iprot.readBitSet(3);
+ java.util.BitSet incoming = iprot.readBitSet(4);
if (incoming.get(0)) {
struct.lowValue = new Date();
struct.lowValue.read(iprot);
@@ -787,6 +901,10 @@ package org.apache.hadoop.hive.metastore.api;
struct.bitVectors = iprot.readBinary();
struct.setBitVectorsIsSet(true);
}
+ if (incoming.get(3)) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ }
}
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DecimalColumnStatsData.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DecimalColumnStatsData.java
index 5ffac52d112..13ab1f691c4 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DecimalColumnStatsData.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DecimalColumnStatsData.java
@@ -16,6 +16,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final org.apache.thrift.protocol.TField NUM_NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("numNulls", org.apache.thrift.protocol.TType.I64, (short)3);
private static final org.apache.thrift.protocol.TField NUM_DVS_FIELD_DESC = new org.apache.thrift.protocol.TField("numDVs", org.apache.thrift.protocol.TType.I64, (short)4);
private static final org.apache.thrift.protocol.TField BIT_VECTORS_FIELD_DESC = new org.apache.thrift.protocol.TField("bitVectors", org.apache.thrift.protocol.TType.STRING, (short)5);
+ private static final org.apache.thrift.protocol.TField HISTOGRAM_FIELD_DESC = new org.apache.thrift.protocol.TField("histogram", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new DecimalColumnStatsDataStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new DecimalColumnStatsDataTupleSchemeFactory();
@@ -25,6 +26,7 @@ package org.apache.hadoop.hive.metastore.api;
private long numNulls; // required
private long numDVs; // required
private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer bitVectors; // optional
+ private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -32,7 +34,8 @@ package org.apache.hadoop.hive.metastore.api;
HIGH_VALUE((short)2, "highValue"),
NUM_NULLS((short)3, "numNulls"),
NUM_DVS((short)4, "numDVs"),
- BIT_VECTORS((short)5, "bitVectors");
+ BIT_VECTORS((short)5, "bitVectors"),
+ HISTOGRAM((short)6, "histogram");
private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
@@ -58,6 +61,8 @@ package org.apache.hadoop.hive.metastore.api;
return NUM_DVS;
case 5: // BIT_VECTORS
return BIT_VECTORS;
+ case 6: // HISTOGRAM
+ return HISTOGRAM;
default:
return null;
}
@@ -102,7 +107,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final int __NUMNULLS_ISSET_ID = 0;
private static final int __NUMDVS_ISSET_ID = 1;
private byte __isset_bitfield = 0;
- private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS};
+ private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS,_Fields.HISTOGRAM};
public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -116,6 +121,8 @@ package org.apache.hadoop.hive.metastore.api;
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
tmpMap.put(_Fields.BIT_VECTORS, new org.apache.thrift.meta_data.FieldMetaData("bitVectors", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
+ tmpMap.put(_Fields.HISTOGRAM, new org.apache.thrift.meta_data.FieldMetaData("histogram", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(DecimalColumnStatsData.class, metaDataMap);
}
@@ -150,6 +157,9 @@ package org.apache.hadoop.hive.metastore.api;
if (other.isSetBitVectors()) {
this.bitVectors = org.apache.thrift.TBaseHelper.copyBinary(other.bitVectors);
}
+ if (other.isSetHistogram()) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(other.histogram);
+ }
}
public DecimalColumnStatsData deepCopy() {
@@ -165,6 +175,7 @@ package org.apache.hadoop.hive.metastore.api;
setNumDVsIsSet(false);
this.numDVs = 0;
this.bitVectors = null;
+ this.histogram = null;
}
@org.apache.thrift.annotation.Nullable
@@ -291,6 +302,38 @@ package org.apache.hadoop.hive.metastore.api;
}
}
+ public byte[] getHistogram() {
+ setHistogram(org.apache.thrift.TBaseHelper.rightSize(histogram));
+ return histogram == null ? null : histogram.array();
+ }
+
+ public java.nio.ByteBuffer bufferForHistogram() {
+ return org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void setHistogram(byte[] histogram) {
+ this.histogram = histogram == null ? (java.nio.ByteBuffer)null : java.nio.ByteBuffer.wrap(histogram.clone());
+ }
+
+ public void setHistogram(@org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void unsetHistogram() {
+ this.histogram = null;
+ }
+
+ /** Returns true if field histogram is set (has been assigned a value) and false otherwise */
+ public boolean isSetHistogram() {
+ return this.histogram != null;
+ }
+
+ public void setHistogramIsSet(boolean value) {
+ if (!value) {
+ this.histogram = null;
+ }
+ }
+
public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
switch (field) {
case LOW_VALUE:
@@ -337,6 +380,18 @@ package org.apache.hadoop.hive.metastore.api;
}
break;
+ case HISTOGRAM:
+ if (value == null) {
+ unsetHistogram();
+ } else {
+ if (value instanceof byte[]) {
+ setHistogram((byte[])value);
+ } else {
+ setHistogram((java.nio.ByteBuffer)value);
+ }
+ }
+ break;
+
}
}
@@ -358,6 +413,9 @@ package org.apache.hadoop.hive.metastore.api;
case BIT_VECTORS:
return getBitVectors();
+ case HISTOGRAM:
+ return getHistogram();
+
}
throw new java.lang.IllegalStateException();
}
@@ -379,6 +437,8 @@ package org.apache.hadoop.hive.metastore.api;
return isSetNumDVs();
case BIT_VECTORS:
return isSetBitVectors();
+ case HISTOGRAM:
+ return isSetHistogram();
}
throw new java.lang.IllegalStateException();
}
@@ -441,6 +501,15 @@ package org.apache.hadoop.hive.metastore.api;
return false;
}
+ boolean this_present_histogram = true && this.isSetHistogram();
+ boolean that_present_histogram = true && that.isSetHistogram();
+ if (this_present_histogram || that_present_histogram) {
+ if (!(this_present_histogram && that_present_histogram))
+ return false;
+ if (!this.histogram.equals(that.histogram))
+ return false;
+ }
+
return true;
}
@@ -464,6 +533,10 @@ package org.apache.hadoop.hive.metastore.api;
if (isSetBitVectors())
hashCode = hashCode * 8191 + bitVectors.hashCode();
+ hashCode = hashCode * 8191 + ((isSetHistogram()) ? 131071 : 524287);
+ if (isSetHistogram())
+ hashCode = hashCode * 8191 + histogram.hashCode();
+
return hashCode;
}
@@ -525,6 +598,16 @@ package org.apache.hadoop.hive.metastore.api;
return lastComparison;
}
}
+ lastComparison = java.lang.Boolean.compare(isSetHistogram(), other.isSetHistogram());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetHistogram()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.histogram, other.histogram);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -583,6 +666,16 @@ package org.apache.hadoop.hive.metastore.api;
}
first = false;
}
+ if (isSetHistogram()) {
+ if (!first) sb.append(", ");
+ sb.append("histogram:");
+ if (this.histogram == null) {
+ sb.append("null");
+ } else {
+ org.apache.thrift.TBaseHelper.toString(this.histogram, sb);
+ }
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -684,6 +777,14 @@ package org.apache.hadoop.hive.metastore.api;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 6: // HISTOGRAM
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -724,6 +825,13 @@ package org.apache.hadoop.hive.metastore.api;
oprot.writeFieldEnd();
}
}
+ if (struct.histogram != null) {
+ if (struct.isSetHistogram()) {
+ oprot.writeFieldBegin(HISTOGRAM_FIELD_DESC);
+ oprot.writeBinary(struct.histogram);
+ oprot.writeFieldEnd();
+ }
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -753,7 +861,10 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
optionals.set(2);
}
- oprot.writeBitSet(optionals, 3);
+ if (struct.isSetHistogram()) {
+ optionals.set(3);
+ }
+ oprot.writeBitSet(optionals, 4);
if (struct.isSetLowValue()) {
struct.lowValue.write(oprot);
}
@@ -763,6 +874,9 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
oprot.writeBinary(struct.bitVectors);
}
+ if (struct.isSetHistogram()) {
+ oprot.writeBinary(struct.histogram);
+ }
}
@Override
@@ -772,7 +886,7 @@ package org.apache.hadoop.hive.metastore.api;
struct.setNumNullsIsSet(true);
struct.numDVs = iprot.readI64();
struct.setNumDVsIsSet(true);
- java.util.BitSet incoming = iprot.readBitSet(3);
+ java.util.BitSet incoming = iprot.readBitSet(4);
if (incoming.get(0)) {
struct.lowValue = new Decimal();
struct.lowValue.read(iprot);
@@ -787,6 +901,10 @@ package org.apache.hadoop.hive.metastore.api;
struct.bitVectors = iprot.readBinary();
struct.setBitVectorsIsSet(true);
}
+ if (incoming.get(3)) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ }
}
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DoubleColumnStatsData.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DoubleColumnStatsData.java
index 75f37150d70..18f9545c8e0 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DoubleColumnStatsData.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DoubleColumnStatsData.java
@@ -16,6 +16,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final org.apache.thrift.protocol.TField NUM_NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("numNulls", org.apache.thrift.protocol.TType.I64, (short)3);
private static final org.apache.thrift.protocol.TField NUM_DVS_FIELD_DESC = new org.apache.thrift.protocol.TField("numDVs", org.apache.thrift.protocol.TType.I64, (short)4);
private static final org.apache.thrift.protocol.TField BIT_VECTORS_FIELD_DESC = new org.apache.thrift.protocol.TField("bitVectors", org.apache.thrift.protocol.TType.STRING, (short)5);
+ private static final org.apache.thrift.protocol.TField HISTOGRAM_FIELD_DESC = new org.apache.thrift.protocol.TField("histogram", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new DoubleColumnStatsDataStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new DoubleColumnStatsDataTupleSchemeFactory();
@@ -25,6 +26,7 @@ package org.apache.hadoop.hive.metastore.api;
private long numNulls; // required
private long numDVs; // required
private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer bitVectors; // optional
+ private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -32,7 +34,8 @@ package org.apache.hadoop.hive.metastore.api;
HIGH_VALUE((short)2, "highValue"),
NUM_NULLS((short)3, "numNulls"),
NUM_DVS((short)4, "numDVs"),
- BIT_VECTORS((short)5, "bitVectors");
+ BIT_VECTORS((short)5, "bitVectors"),
+ HISTOGRAM((short)6, "histogram");
private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
@@ -58,6 +61,8 @@ package org.apache.hadoop.hive.metastore.api;
return NUM_DVS;
case 5: // BIT_VECTORS
return BIT_VECTORS;
+ case 6: // HISTOGRAM
+ return HISTOGRAM;
default:
return null;
}
@@ -104,7 +109,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final int __NUMNULLS_ISSET_ID = 2;
private static final int __NUMDVS_ISSET_ID = 3;
private byte __isset_bitfield = 0;
- private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS};
+ private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS,_Fields.HISTOGRAM};
public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -118,6 +123,8 @@ package org.apache.hadoop.hive.metastore.api;
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
tmpMap.put(_Fields.BIT_VECTORS, new org.apache.thrift.meta_data.FieldMetaData("bitVectors", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
+ tmpMap.put(_Fields.HISTOGRAM, new org.apache.thrift.meta_data.FieldMetaData("histogram", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(DoubleColumnStatsData.class, metaDataMap);
}
@@ -148,6 +155,9 @@ package org.apache.hadoop.hive.metastore.api;
if (other.isSetBitVectors()) {
this.bitVectors = org.apache.thrift.TBaseHelper.copyBinary(other.bitVectors);
}
+ if (other.isSetHistogram()) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(other.histogram);
+ }
}
public DoubleColumnStatsData deepCopy() {
@@ -165,6 +175,7 @@ package org.apache.hadoop.hive.metastore.api;
setNumDVsIsSet(false);
this.numDVs = 0;
this.bitVectors = null;
+ this.histogram = null;
}
public double getLowValue() {
@@ -287,6 +298,38 @@ package org.apache.hadoop.hive.metastore.api;
}
}
+ public byte[] getHistogram() {
+ setHistogram(org.apache.thrift.TBaseHelper.rightSize(histogram));
+ return histogram == null ? null : histogram.array();
+ }
+
+ public java.nio.ByteBuffer bufferForHistogram() {
+ return org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void setHistogram(byte[] histogram) {
+ this.histogram = histogram == null ? (java.nio.ByteBuffer)null : java.nio.ByteBuffer.wrap(histogram.clone());
+ }
+
+ public void setHistogram(@org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void unsetHistogram() {
+ this.histogram = null;
+ }
+
+ /** Returns true if field histogram is set (has been assigned a value) and false otherwise */
+ public boolean isSetHistogram() {
+ return this.histogram != null;
+ }
+
+ public void setHistogramIsSet(boolean value) {
+ if (!value) {
+ this.histogram = null;
+ }
+ }
+
public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
switch (field) {
case LOW_VALUE:
@@ -333,6 +376,18 @@ package org.apache.hadoop.hive.metastore.api;
}
break;
+ case HISTOGRAM:
+ if (value == null) {
+ unsetHistogram();
+ } else {
+ if (value instanceof byte[]) {
+ setHistogram((byte[])value);
+ } else {
+ setHistogram((java.nio.ByteBuffer)value);
+ }
+ }
+ break;
+
}
}
@@ -354,6 +409,9 @@ package org.apache.hadoop.hive.metastore.api;
case BIT_VECTORS:
return getBitVectors();
+ case HISTOGRAM:
+ return getHistogram();
+
}
throw new java.lang.IllegalStateException();
}
@@ -375,6 +433,8 @@ package org.apache.hadoop.hive.metastore.api;
return isSetNumDVs();
case BIT_VECTORS:
return isSetBitVectors();
+ case HISTOGRAM:
+ return isSetHistogram();
}
throw new java.lang.IllegalStateException();
}
@@ -437,6 +497,15 @@ package org.apache.hadoop.hive.metastore.api;
return false;
}
+ boolean this_present_histogram = true && this.isSetHistogram();
+ boolean that_present_histogram = true && that.isSetHistogram();
+ if (this_present_histogram || that_present_histogram) {
+ if (!(this_present_histogram && that_present_histogram))
+ return false;
+ if (!this.histogram.equals(that.histogram))
+ return false;
+ }
+
return true;
}
@@ -460,6 +529,10 @@ package org.apache.hadoop.hive.metastore.api;
if (isSetBitVectors())
hashCode = hashCode * 8191 + bitVectors.hashCode();
+ hashCode = hashCode * 8191 + ((isSetHistogram()) ? 131071 : 524287);
+ if (isSetHistogram())
+ hashCode = hashCode * 8191 + histogram.hashCode();
+
return hashCode;
}
@@ -521,6 +594,16 @@ package org.apache.hadoop.hive.metastore.api;
return lastComparison;
}
}
+ lastComparison = java.lang.Boolean.compare(isSetHistogram(), other.isSetHistogram());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetHistogram()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.histogram, other.histogram);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -571,6 +654,16 @@ package org.apache.hadoop.hive.metastore.api;
}
first = false;
}
+ if (isSetHistogram()) {
+ if (!first) sb.append(", ");
+ sb.append("histogram:");
+ if (this.histogram == null) {
+ sb.append("null");
+ } else {
+ org.apache.thrift.TBaseHelper.toString(this.histogram, sb);
+ }
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -664,6 +757,14 @@ package org.apache.hadoop.hive.metastore.api;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 6: // HISTOGRAM
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -700,6 +801,13 @@ package org.apache.hadoop.hive.metastore.api;
oprot.writeFieldEnd();
}
}
+ if (struct.histogram != null) {
+ if (struct.isSetHistogram()) {
+ oprot.writeFieldBegin(HISTOGRAM_FIELD_DESC);
+ oprot.writeBinary(struct.histogram);
+ oprot.writeFieldEnd();
+ }
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -729,7 +837,10 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
optionals.set(2);
}
- oprot.writeBitSet(optionals, 3);
+ if (struct.isSetHistogram()) {
+ optionals.set(3);
+ }
+ oprot.writeBitSet(optionals, 4);
if (struct.isSetLowValue()) {
oprot.writeDouble(struct.lowValue);
}
@@ -739,6 +850,9 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
oprot.writeBinary(struct.bitVectors);
}
+ if (struct.isSetHistogram()) {
+ oprot.writeBinary(struct.histogram);
+ }
}
@Override
@@ -748,7 +862,7 @@ package org.apache.hadoop.hive.metastore.api;
struct.setNumNullsIsSet(true);
struct.numDVs = iprot.readI64();
struct.setNumDVsIsSet(true);
- java.util.BitSet incoming = iprot.readBitSet(3);
+ java.util.BitSet incoming = iprot.readBitSet(4);
if (incoming.get(0)) {
struct.lowValue = iprot.readDouble();
struct.setLowValueIsSet(true);
@@ -761,6 +875,10 @@ package org.apache.hadoop.hive.metastore.api;
struct.bitVectors = iprot.readBinary();
struct.setBitVectorsIsSet(true);
}
+ if (incoming.get(3)) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ }
}
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LongColumnStatsData.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LongColumnStatsData.java
index 9c9da891227..6c7be430a52 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LongColumnStatsData.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LongColumnStatsData.java
@@ -16,6 +16,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final org.apache.thrift.protocol.TField NUM_NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("numNulls", org.apache.thrift.protocol.TType.I64, (short)3);
private static final org.apache.thrift.protocol.TField NUM_DVS_FIELD_DESC = new org.apache.thrift.protocol.TField("numDVs", org.apache.thrift.protocol.TType.I64, (short)4);
private static final org.apache.thrift.protocol.TField BIT_VECTORS_FIELD_DESC = new org.apache.thrift.protocol.TField("bitVectors", org.apache.thrift.protocol.TType.STRING, (short)5);
+ private static final org.apache.thrift.protocol.TField HISTOGRAM_FIELD_DESC = new org.apache.thrift.protocol.TField("histogram", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new LongColumnStatsDataStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new LongColumnStatsDataTupleSchemeFactory();
@@ -25,6 +26,7 @@ package org.apache.hadoop.hive.metastore.api;
private long numNulls; // required
private long numDVs; // required
private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer bitVectors; // optional
+ private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -32,7 +34,8 @@ package org.apache.hadoop.hive.metastore.api;
HIGH_VALUE((short)2, "highValue"),
NUM_NULLS((short)3, "numNulls"),
NUM_DVS((short)4, "numDVs"),
- BIT_VECTORS((short)5, "bitVectors");
+ BIT_VECTORS((short)5, "bitVectors"),
+ HISTOGRAM((short)6, "histogram");
private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
@@ -58,6 +61,8 @@ package org.apache.hadoop.hive.metastore.api;
return NUM_DVS;
case 5: // BIT_VECTORS
return BIT_VECTORS;
+ case 6: // HISTOGRAM
+ return HISTOGRAM;
default:
return null;
}
@@ -104,7 +109,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final int __NUMNULLS_ISSET_ID = 2;
private static final int __NUMDVS_ISSET_ID = 3;
private byte __isset_bitfield = 0;
- private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS};
+ private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS,_Fields.HISTOGRAM};
public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -118,6 +123,8 @@ package org.apache.hadoop.hive.metastore.api;
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
tmpMap.put(_Fields.BIT_VECTORS, new org.apache.thrift.meta_data.FieldMetaData("bitVectors", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
+ tmpMap.put(_Fields.HISTOGRAM, new org.apache.thrift.meta_data.FieldMetaData("histogram", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(LongColumnStatsData.class, metaDataMap);
}
@@ -148,6 +155,9 @@ package org.apache.hadoop.hive.metastore.api;
if (other.isSetBitVectors()) {
this.bitVectors = org.apache.thrift.TBaseHelper.copyBinary(other.bitVectors);
}
+ if (other.isSetHistogram()) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(other.histogram);
+ }
}
public LongColumnStatsData deepCopy() {
@@ -165,6 +175,7 @@ package org.apache.hadoop.hive.metastore.api;
setNumDVsIsSet(false);
this.numDVs = 0;
this.bitVectors = null;
+ this.histogram = null;
}
public long getLowValue() {
@@ -287,6 +298,38 @@ package org.apache.hadoop.hive.metastore.api;
}
}
+ public byte[] getHistogram() {
+ setHistogram(org.apache.thrift.TBaseHelper.rightSize(histogram));
+ return histogram == null ? null : histogram.array();
+ }
+
+ public java.nio.ByteBuffer bufferForHistogram() {
+ return org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void setHistogram(byte[] histogram) {
+ this.histogram = histogram == null ? (java.nio.ByteBuffer)null : java.nio.ByteBuffer.wrap(histogram.clone());
+ }
+
+ public void setHistogram(@org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void unsetHistogram() {
+ this.histogram = null;
+ }
+
+ /** Returns true if field histogram is set (has been assigned a value) and false otherwise */
+ public boolean isSetHistogram() {
+ return this.histogram != null;
+ }
+
+ public void setHistogramIsSet(boolean value) {
+ if (!value) {
+ this.histogram = null;
+ }
+ }
+
public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
switch (field) {
case LOW_VALUE:
@@ -333,6 +376,18 @@ package org.apache.hadoop.hive.metastore.api;
}
break;
+ case HISTOGRAM:
+ if (value == null) {
+ unsetHistogram();
+ } else {
+ if (value instanceof byte[]) {
+ setHistogram((byte[])value);
+ } else {
+ setHistogram((java.nio.ByteBuffer)value);
+ }
+ }
+ break;
+
}
}
@@ -354,6 +409,9 @@ package org.apache.hadoop.hive.metastore.api;
case BIT_VECTORS:
return getBitVectors();
+ case HISTOGRAM:
+ return getHistogram();
+
}
throw new java.lang.IllegalStateException();
}
@@ -375,6 +433,8 @@ package org.apache.hadoop.hive.metastore.api;
return isSetNumDVs();
case BIT_VECTORS:
return isSetBitVectors();
+ case HISTOGRAM:
+ return isSetHistogram();
}
throw new java.lang.IllegalStateException();
}
@@ -437,6 +497,15 @@ package org.apache.hadoop.hive.metastore.api;
return false;
}
+ boolean this_present_histogram = true && this.isSetHistogram();
+ boolean that_present_histogram = true && that.isSetHistogram();
+ if (this_present_histogram || that_present_histogram) {
+ if (!(this_present_histogram && that_present_histogram))
+ return false;
+ if (!this.histogram.equals(that.histogram))
+ return false;
+ }
+
return true;
}
@@ -460,6 +529,10 @@ package org.apache.hadoop.hive.metastore.api;
if (isSetBitVectors())
hashCode = hashCode * 8191 + bitVectors.hashCode();
+ hashCode = hashCode * 8191 + ((isSetHistogram()) ? 131071 : 524287);
+ if (isSetHistogram())
+ hashCode = hashCode * 8191 + histogram.hashCode();
+
return hashCode;
}
@@ -521,6 +594,16 @@ package org.apache.hadoop.hive.metastore.api;
return lastComparison;
}
}
+ lastComparison = java.lang.Boolean.compare(isSetHistogram(), other.isSetHistogram());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetHistogram()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.histogram, other.histogram);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -571,6 +654,16 @@ package org.apache.hadoop.hive.metastore.api;
}
first = false;
}
+ if (isSetHistogram()) {
+ if (!first) sb.append(", ");
+ sb.append("histogram:");
+ if (this.histogram == null) {
+ sb.append("null");
+ } else {
+ org.apache.thrift.TBaseHelper.toString(this.histogram, sb);
+ }
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -664,6 +757,14 @@ package org.apache.hadoop.hive.metastore.api;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 6: // HISTOGRAM
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -700,6 +801,13 @@ package org.apache.hadoop.hive.metastore.api;
oprot.writeFieldEnd();
}
}
+ if (struct.histogram != null) {
+ if (struct.isSetHistogram()) {
+ oprot.writeFieldBegin(HISTOGRAM_FIELD_DESC);
+ oprot.writeBinary(struct.histogram);
+ oprot.writeFieldEnd();
+ }
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -729,7 +837,10 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
optionals.set(2);
}
- oprot.writeBitSet(optionals, 3);
+ if (struct.isSetHistogram()) {
+ optionals.set(3);
+ }
+ oprot.writeBitSet(optionals, 4);
if (struct.isSetLowValue()) {
oprot.writeI64(struct.lowValue);
}
@@ -739,6 +850,9 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
oprot.writeBinary(struct.bitVectors);
}
+ if (struct.isSetHistogram()) {
+ oprot.writeBinary(struct.histogram);
+ }
}
@Override
@@ -748,7 +862,7 @@ package org.apache.hadoop.hive.metastore.api;
struct.setNumNullsIsSet(true);
struct.numDVs = iprot.readI64();
struct.setNumDVsIsSet(true);
- java.util.BitSet incoming = iprot.readBitSet(3);
+ java.util.BitSet incoming = iprot.readBitSet(4);
if (incoming.get(0)) {
struct.lowValue = iprot.readI64();
struct.setLowValueIsSet(true);
@@ -761,6 +875,10 @@ package org.apache.hadoop.hive.metastore.api;
struct.bitVectors = iprot.readBinary();
struct.setBitVectorsIsSet(true);
}
+ if (incoming.get(3)) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ }
}
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TimestampColumnStatsData.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TimestampColumnStatsData.java
index 64611b78c9e..0fc71367920 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TimestampColumnStatsData.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TimestampColumnStatsData.java
@@ -16,6 +16,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final org.apache.thrift.protocol.TField NUM_NULLS_FIELD_DESC = new org.apache.thrift.protocol.TField("numNulls", org.apache.thrift.protocol.TType.I64, (short)3);
private static final org.apache.thrift.protocol.TField NUM_DVS_FIELD_DESC = new org.apache.thrift.protocol.TField("numDVs", org.apache.thrift.protocol.TType.I64, (short)4);
private static final org.apache.thrift.protocol.TField BIT_VECTORS_FIELD_DESC = new org.apache.thrift.protocol.TField("bitVectors", org.apache.thrift.protocol.TType.STRING, (short)5);
+ private static final org.apache.thrift.protocol.TField HISTOGRAM_FIELD_DESC = new org.apache.thrift.protocol.TField("histogram", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new TimestampColumnStatsDataStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new TimestampColumnStatsDataTupleSchemeFactory();
@@ -25,6 +26,7 @@ package org.apache.hadoop.hive.metastore.api;
private long numNulls; // required
private long numDVs; // required
private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer bitVectors; // optional
+ private @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -32,7 +34,8 @@ package org.apache.hadoop.hive.metastore.api;
HIGH_VALUE((short)2, "highValue"),
NUM_NULLS((short)3, "numNulls"),
NUM_DVS((short)4, "numDVs"),
- BIT_VECTORS((short)5, "bitVectors");
+ BIT_VECTORS((short)5, "bitVectors"),
+ HISTOGRAM((short)6, "histogram");
private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
@@ -58,6 +61,8 @@ package org.apache.hadoop.hive.metastore.api;
return NUM_DVS;
case 5: // BIT_VECTORS
return BIT_VECTORS;
+ case 6: // HISTOGRAM
+ return HISTOGRAM;
default:
return null;
}
@@ -102,7 +107,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final int __NUMNULLS_ISSET_ID = 0;
private static final int __NUMDVS_ISSET_ID = 1;
private byte __isset_bitfield = 0;
- private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS};
+ private static final _Fields optionals[] = {_Fields.LOW_VALUE,_Fields.HIGH_VALUE,_Fields.BIT_VECTORS,_Fields.HISTOGRAM};
public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -116,6 +121,8 @@ package org.apache.hadoop.hive.metastore.api;
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
tmpMap.put(_Fields.BIT_VECTORS, new org.apache.thrift.meta_data.FieldMetaData("bitVectors", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
+ tmpMap.put(_Fields.HISTOGRAM, new org.apache.thrift.meta_data.FieldMetaData("histogram", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TimestampColumnStatsData.class, metaDataMap);
}
@@ -150,6 +157,9 @@ package org.apache.hadoop.hive.metastore.api;
if (other.isSetBitVectors()) {
this.bitVectors = org.apache.thrift.TBaseHelper.copyBinary(other.bitVectors);
}
+ if (other.isSetHistogram()) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(other.histogram);
+ }
}
public TimestampColumnStatsData deepCopy() {
@@ -165,6 +175,7 @@ package org.apache.hadoop.hive.metastore.api;
setNumDVsIsSet(false);
this.numDVs = 0;
this.bitVectors = null;
+ this.histogram = null;
}
@org.apache.thrift.annotation.Nullable
@@ -291,6 +302,38 @@ package org.apache.hadoop.hive.metastore.api;
}
}
+ public byte[] getHistogram() {
+ setHistogram(org.apache.thrift.TBaseHelper.rightSize(histogram));
+ return histogram == null ? null : histogram.array();
+ }
+
+ public java.nio.ByteBuffer bufferForHistogram() {
+ return org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void setHistogram(byte[] histogram) {
+ this.histogram = histogram == null ? (java.nio.ByteBuffer)null : java.nio.ByteBuffer.wrap(histogram.clone());
+ }
+
+ public void setHistogram(@org.apache.thrift.annotation.Nullable java.nio.ByteBuffer histogram) {
+ this.histogram = org.apache.thrift.TBaseHelper.copyBinary(histogram);
+ }
+
+ public void unsetHistogram() {
+ this.histogram = null;
+ }
+
+ /** Returns true if field histogram is set (has been assigned a value) and false otherwise */
+ public boolean isSetHistogram() {
+ return this.histogram != null;
+ }
+
+ public void setHistogramIsSet(boolean value) {
+ if (!value) {
+ this.histogram = null;
+ }
+ }
+
public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) {
switch (field) {
case LOW_VALUE:
@@ -337,6 +380,18 @@ package org.apache.hadoop.hive.metastore.api;
}
break;
+ case HISTOGRAM:
+ if (value == null) {
+ unsetHistogram();
+ } else {
+ if (value instanceof byte[]) {
+ setHistogram((byte[])value);
+ } else {
+ setHistogram((java.nio.ByteBuffer)value);
+ }
+ }
+ break;
+
}
}
@@ -358,6 +413,9 @@ package org.apache.hadoop.hive.metastore.api;
case BIT_VECTORS:
return getBitVectors();
+ case HISTOGRAM:
+ return getHistogram();
+
}
throw new java.lang.IllegalStateException();
}
@@ -379,6 +437,8 @@ package org.apache.hadoop.hive.metastore.api;
return isSetNumDVs();
case BIT_VECTORS:
return isSetBitVectors();
+ case HISTOGRAM:
+ return isSetHistogram();
}
throw new java.lang.IllegalStateException();
}
@@ -441,6 +501,15 @@ package org.apache.hadoop.hive.metastore.api;
return false;
}
+ boolean this_present_histogram = true && this.isSetHistogram();
+ boolean that_present_histogram = true && that.isSetHistogram();
+ if (this_present_histogram || that_present_histogram) {
+ if (!(this_present_histogram && that_present_histogram))
+ return false;
+ if (!this.histogram.equals(that.histogram))
+ return false;
+ }
+
return true;
}
@@ -464,6 +533,10 @@ package org.apache.hadoop.hive.metastore.api;
if (isSetBitVectors())
hashCode = hashCode * 8191 + bitVectors.hashCode();
+ hashCode = hashCode * 8191 + ((isSetHistogram()) ? 131071 : 524287);
+ if (isSetHistogram())
+ hashCode = hashCode * 8191 + histogram.hashCode();
+
return hashCode;
}
@@ -525,6 +598,16 @@ package org.apache.hadoop.hive.metastore.api;
return lastComparison;
}
}
+ lastComparison = java.lang.Boolean.compare(isSetHistogram(), other.isSetHistogram());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetHistogram()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.histogram, other.histogram);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -583,6 +666,16 @@ package org.apache.hadoop.hive.metastore.api;
}
first = false;
}
+ if (isSetHistogram()) {
+ if (!first) sb.append(", ");
+ sb.append("histogram:");
+ if (this.histogram == null) {
+ sb.append("null");
+ } else {
+ org.apache.thrift.TBaseHelper.toString(this.histogram, sb);
+ }
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -684,6 +777,14 @@ package org.apache.hadoop.hive.metastore.api;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 6: // HISTOGRAM
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -724,6 +825,13 @@ package org.apache.hadoop.hive.metastore.api;
oprot.writeFieldEnd();
}
}
+ if (struct.histogram != null) {
+ if (struct.isSetHistogram()) {
+ oprot.writeFieldBegin(HISTOGRAM_FIELD_DESC);
+ oprot.writeBinary(struct.histogram);
+ oprot.writeFieldEnd();
+ }
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -753,7 +861,10 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
optionals.set(2);
}
- oprot.writeBitSet(optionals, 3);
+ if (struct.isSetHistogram()) {
+ optionals.set(3);
+ }
+ oprot.writeBitSet(optionals, 4);
if (struct.isSetLowValue()) {
struct.lowValue.write(oprot);
}
@@ -763,6 +874,9 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetBitVectors()) {
oprot.writeBinary(struct.bitVectors);
}
+ if (struct.isSetHistogram()) {
+ oprot.writeBinary(struct.histogram);
+ }
}
@Override
@@ -772,7 +886,7 @@ package org.apache.hadoop.hive.metastore.api;
struct.setNumNullsIsSet(true);
struct.numDVs = iprot.readI64();
struct.setNumDVsIsSet(true);
- java.util.BitSet incoming = iprot.readBitSet(3);
+ java.util.BitSet incoming = iprot.readBitSet(4);
if (incoming.get(0)) {
struct.lowValue = new Timestamp();
struct.lowValue.read(iprot);
@@ -787,6 +901,10 @@ package org.apache.hadoop.hive.metastore.api;
struct.bitVectors = iprot.readBinary();
struct.setBitVectorsIsSet(true);
}
+ if (incoming.get(3)) {
+ struct.histogram = iprot.readBinary();
+ struct.setHistogramIsSet(true);
+ }
}
}
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DateColumnStatsData.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DateColumnStatsData.php
index 14eb6b794e6..49607f8565e 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DateColumnStatsData.php
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DateColumnStatsData.php
@@ -48,6 +48,11 @@ class DateColumnStatsData
'isRequired' => false,
'type' => TType::STRING,
),
+ 6 => array(
+ 'var' => 'histogram',
+ 'isRequired' => false,
+ 'type' => TType::STRING,
+ ),
);
/**
@@ -70,6 +75,10 @@ class DateColumnStatsData
* @var string
*/
public $bitVectors = null;
+ /**
+ * @var string
+ */
+ public $histogram = null;
public function __construct($vals = null)
{
@@ -89,6 +98,9 @@ class DateColumnStatsData
if (isset($vals['bitVectors'])) {
$this->bitVectors = $vals['bitVectors'];
}
+ if (isset($vals['histogram'])) {
+ $this->histogram = $vals['histogram'];
+ }
}
}
@@ -148,6 +160,13 @@ class DateColumnStatsData
$xfer += $input->skip($ftype);
}
break;
+ case 6:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->histogram);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -193,6 +212,11 @@ class DateColumnStatsData
$xfer += $output->writeString($this->bitVectors);
$xfer += $output->writeFieldEnd();
}
+ if ($this->histogram !== null) {
+ $xfer += $output->writeFieldBegin('histogram', TType::STRING, 6);
+ $xfer += $output->writeString($this->histogram);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DecimalColumnStatsData.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DecimalColumnStatsData.php
index 54bee3ce4d9..94c94cc3dd9 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DecimalColumnStatsData.php
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DecimalColumnStatsData.php
@@ -48,6 +48,11 @@ class DecimalColumnStatsData
'isRequired' => false,
'type' => TType::STRING,
),
+ 6 => array(
+ 'var' => 'histogram',
+ 'isRequired' => false,
+ 'type' => TType::STRING,
+ ),
);
/**
@@ -70,6 +75,10 @@ class DecimalColumnStatsData
* @var string
*/
public $bitVectors = null;
+ /**
+ * @var string
+ */
+ public $histogram = null;
public function __construct($vals = null)
{
@@ -89,6 +98,9 @@ class DecimalColumnStatsData
if (isset($vals['bitVectors'])) {
$this->bitVectors = $vals['bitVectors'];
}
+ if (isset($vals['histogram'])) {
+ $this->histogram = $vals['histogram'];
+ }
}
}
@@ -148,6 +160,13 @@ class DecimalColumnStatsData
$xfer += $input->skip($ftype);
}
break;
+ case 6:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->histogram);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -193,6 +212,11 @@ class DecimalColumnStatsData
$xfer += $output->writeString($this->bitVectors);
$xfer += $output->writeFieldEnd();
}
+ if ($this->histogram !== null) {
+ $xfer += $output->writeFieldBegin('histogram', TType::STRING, 6);
+ $xfer += $output->writeString($this->histogram);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DoubleColumnStatsData.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DoubleColumnStatsData.php
index 0df8dfc672a..cb18d828258 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DoubleColumnStatsData.php
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/DoubleColumnStatsData.php
@@ -46,6 +46,11 @@ class DoubleColumnStatsData
'isRequired' => false,
'type' => TType::STRING,
),
+ 6 => array(
+ 'var' => 'histogram',
+ 'isRequired' => false,
+ 'type' => TType::STRING,
+ ),
);
/**
@@ -68,6 +73,10 @@ class DoubleColumnStatsData
* @var string
*/
public $bitVectors = null;
+ /**
+ * @var string
+ */
+ public $histogram = null;
public function __construct($vals = null)
{
@@ -87,6 +96,9 @@ class DoubleColumnStatsData
if (isset($vals['bitVectors'])) {
$this->bitVectors = $vals['bitVectors'];
}
+ if (isset($vals['histogram'])) {
+ $this->histogram = $vals['histogram'];
+ }
}
}
@@ -144,6 +156,13 @@ class DoubleColumnStatsData
$xfer += $input->skip($ftype);
}
break;
+ case 6:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->histogram);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -183,6 +202,11 @@ class DoubleColumnStatsData
$xfer += $output->writeString($this->bitVectors);
$xfer += $output->writeFieldEnd();
}
+ if ($this->histogram !== null) {
+ $xfer += $output->writeFieldBegin('histogram', TType::STRING, 6);
+ $xfer += $output->writeString($this->histogram);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/LongColumnStatsData.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/LongColumnStatsData.php
index f90f53eb37b..5dd2d8320e3 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/LongColumnStatsData.php
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/LongColumnStatsData.php
@@ -46,6 +46,11 @@ class LongColumnStatsData
'isRequired' => false,
'type' => TType::STRING,
),
+ 6 => array(
+ 'var' => 'histogram',
+ 'isRequired' => false,
+ 'type' => TType::STRING,
+ ),
);
/**
@@ -68,6 +73,10 @@ class LongColumnStatsData
* @var string
*/
public $bitVectors = null;
+ /**
+ * @var string
+ */
+ public $histogram = null;
public function __construct($vals = null)
{
@@ -87,6 +96,9 @@ class LongColumnStatsData
if (isset($vals['bitVectors'])) {
$this->bitVectors = $vals['bitVectors'];
}
+ if (isset($vals['histogram'])) {
+ $this->histogram = $vals['histogram'];
+ }
}
}
@@ -144,6 +156,13 @@ class LongColumnStatsData
$xfer += $input->skip($ftype);
}
break;
+ case 6:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->histogram);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -183,6 +202,11 @@ class LongColumnStatsData
$xfer += $output->writeString($this->bitVectors);
$xfer += $output->writeFieldEnd();
}
+ if ($this->histogram !== null) {
+ $xfer += $output->writeFieldBegin('histogram', TType::STRING, 6);
+ $xfer += $output->writeString($this->histogram);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/TimestampColumnStatsData.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/TimestampColumnStatsData.php
index 25f11b593ba..2ce03a68513 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/TimestampColumnStatsData.php
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/TimestampColumnStatsData.php
@@ -48,6 +48,11 @@ class TimestampColumnStatsData
'isRequired' => false,
'type' => TType::STRING,
),
+ 6 => array(
+ 'var' => 'histogram',
+ 'isRequired' => false,
+ 'type' => TType::STRING,
+ ),
);
/**
@@ -70,6 +75,10 @@ class TimestampColumnStatsData
* @var string
*/
public $bitVectors = null;
+ /**
+ * @var string
+ */
+ public $histogram = null;
public function __construct($vals = null)
{
@@ -89,6 +98,9 @@ class TimestampColumnStatsData
if (isset($vals['bitVectors'])) {
$this->bitVectors = $vals['bitVectors'];
}
+ if (isset($vals['histogram'])) {
+ $this->histogram = $vals['histogram'];
+ }
}
}
@@ -148,6 +160,13 @@ class TimestampColumnStatsData
$xfer += $input->skip($ftype);
}
break;
+ case 6:
+ if ($ftype == TType::STRING) {
+ $xfer += $input->readString($this->histogram);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -193,6 +212,11 @@ class TimestampColumnStatsData
$xfer += $output->writeString($this->bitVectors);
$xfer += $output->writeFieldEnd();
}
+ if ($this->histogram !== null) {
+ $xfer += $output->writeFieldBegin('histogram', TType::STRING, 6);
+ $xfer += $output->writeString($this->histogram);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index 57ceaf231e7..e67dc9c9efb 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -4882,16 +4882,18 @@ class DoubleColumnStatsData(object):
- numNulls
- numDVs
- bitVectors
+ - histogram
"""
- def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None,):
+ def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None, histogram=None,):
self.lowValue = lowValue
self.highValue = highValue
self.numNulls = numNulls
self.numDVs = numDVs
self.bitVectors = bitVectors
+ self.histogram = histogram
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
@@ -4927,6 +4929,11 @@ class DoubleColumnStatsData(object):
self.bitVectors = iprot.readBinary()
else:
iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.STRING:
+ self.histogram = iprot.readBinary()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -4957,6 +4964,10 @@ class DoubleColumnStatsData(object):
oprot.writeFieldBegin('bitVectors', TType.STRING, 5)
oprot.writeBinary(self.bitVectors)
oprot.writeFieldEnd()
+ if self.histogram is not None:
+ oprot.writeFieldBegin('histogram', TType.STRING, 6)
+ oprot.writeBinary(self.histogram)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -4987,16 +4998,18 @@ class LongColumnStatsData(object):
- numNulls
- numDVs
- bitVectors
+ - histogram
"""
- def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None,):
+ def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None, histogram=None,):
self.lowValue = lowValue
self.highValue = highValue
self.numNulls = numNulls
self.numDVs = numDVs
self.bitVectors = bitVectors
+ self.histogram = histogram
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
@@ -5032,6 +5045,11 @@ class LongColumnStatsData(object):
self.bitVectors = iprot.readBinary()
else:
iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.STRING:
+ self.histogram = iprot.readBinary()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -5062,6 +5080,10 @@ class LongColumnStatsData(object):
oprot.writeFieldBegin('bitVectors', TType.STRING, 5)
oprot.writeBinary(self.bitVectors)
oprot.writeFieldEnd()
+ if self.histogram is not None:
+ oprot.writeFieldBegin('histogram', TType.STRING, 6)
+ oprot.writeBinary(self.histogram)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -5369,16 +5391,18 @@ class DecimalColumnStatsData(object):
- numNulls
- numDVs
- bitVectors
+ - histogram
"""
- def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None,):
+ def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None, histogram=None,):
self.lowValue = lowValue
self.highValue = highValue
self.numNulls = numNulls
self.numDVs = numDVs
self.bitVectors = bitVectors
+ self.histogram = histogram
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
@@ -5416,6 +5440,11 @@ class DecimalColumnStatsData(object):
self.bitVectors = iprot.readBinary()
else:
iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.STRING:
+ self.histogram = iprot.readBinary()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -5446,6 +5475,10 @@ class DecimalColumnStatsData(object):
oprot.writeFieldBegin('bitVectors', TType.STRING, 5)
oprot.writeBinary(self.bitVectors)
oprot.writeFieldEnd()
+ if self.histogram is not None:
+ oprot.writeFieldBegin('histogram', TType.STRING, 6)
+ oprot.writeBinary(self.histogram)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -5535,16 +5568,18 @@ class DateColumnStatsData(object):
- numNulls
- numDVs
- bitVectors
+ - histogram
"""
- def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None,):
+ def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None, histogram=None,):
self.lowValue = lowValue
self.highValue = highValue
self.numNulls = numNulls
self.numDVs = numDVs
self.bitVectors = bitVectors
+ self.histogram = histogram
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
@@ -5582,6 +5617,11 @@ class DateColumnStatsData(object):
self.bitVectors = iprot.readBinary()
else:
iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.STRING:
+ self.histogram = iprot.readBinary()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -5612,6 +5652,10 @@ class DateColumnStatsData(object):
oprot.writeFieldBegin('bitVectors', TType.STRING, 5)
oprot.writeBinary(self.bitVectors)
oprot.writeFieldEnd()
+ if self.histogram is not None:
+ oprot.writeFieldBegin('histogram', TType.STRING, 6)
+ oprot.writeBinary(self.histogram)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -5701,16 +5745,18 @@ class TimestampColumnStatsData(object):
- numNulls
- numDVs
- bitVectors
+ - histogram
"""
- def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None,):
+ def __init__(self, lowValue=None, highValue=None, numNulls=None, numDVs=None, bitVectors=None, histogram=None,):
self.lowValue = lowValue
self.highValue = highValue
self.numNulls = numNulls
self.numDVs = numDVs
self.bitVectors = bitVectors
+ self.histogram = histogram
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
@@ -5748,6 +5794,11 @@ class TimestampColumnStatsData(object):
self.bitVectors = iprot.readBinary()
else:
iprot.skip(ftype)
+ elif fid == 6:
+ if ftype == TType.STRING:
+ self.histogram = iprot.readBinary()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -5778,6 +5829,10 @@ class TimestampColumnStatsData(object):
oprot.writeFieldBegin('bitVectors', TType.STRING, 5)
oprot.writeBinary(self.bitVectors)
oprot.writeFieldEnd()
+ if self.histogram is not None:
+ oprot.writeFieldBegin('histogram', TType.STRING, 6)
+ oprot.writeBinary(self.histogram)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -30299,6 +30354,7 @@ DoubleColumnStatsData.thrift_spec = (
(3, TType.I64, 'numNulls', None, None, ), # 3
(4, TType.I64, 'numDVs', None, None, ), # 4
(5, TType.STRING, 'bitVectors', 'BINARY', None, ), # 5
+ (6, TType.STRING, 'histogram', 'BINARY', None, ), # 6
)
all_structs.append(LongColumnStatsData)
LongColumnStatsData.thrift_spec = (
@@ -30308,6 +30364,7 @@ LongColumnStatsData.thrift_spec = (
(3, TType.I64, 'numNulls', None, None, ), # 3
(4, TType.I64, 'numDVs', None, None, ), # 4
(5, TType.STRING, 'bitVectors', 'BINARY', None, ), # 5
+ (6, TType.STRING, 'histogram', 'BINARY', None, ), # 6
)
all_structs.append(StringColumnStatsData)
StringColumnStatsData.thrift_spec = (
@@ -30341,6 +30398,7 @@ DecimalColumnStatsData.thrift_spec = (
(3, TType.I64, 'numNulls', None, None, ), # 3
(4, TType.I64, 'numDVs', None, None, ), # 4
(5, TType.STRING, 'bitVectors', 'BINARY', None, ), # 5
+ (6, TType.STRING, 'histogram', 'BINARY', None, ), # 6
)
all_structs.append(Date)
Date.thrift_spec = (
@@ -30355,6 +30413,7 @@ DateColumnStatsData.thrift_spec = (
(3, TType.I64, 'numNulls', None, None, ), # 3
(4, TType.I64, 'numDVs', None, None, ), # 4
(5, TType.STRING, 'bitVectors', 'BINARY', None, ), # 5
+ (6, TType.STRING, 'histogram', 'BINARY', None, ), # 6
)
all_structs.append(Timestamp)
Timestamp.thrift_spec = (
@@ -30369,6 +30428,7 @@ TimestampColumnStatsData.thrift_spec = (
(3, TType.I64, 'numNulls', None, None, ), # 3
(4, TType.I64, 'numDVs', None, None, ), # 4
(5, TType.STRING, 'bitVectors', 'BINARY', None, ), # 5
+ (6, TType.STRING, 'histogram', 'BINARY', None, ), # 6
)
all_structs.append(ColumnStatisticsData)
ColumnStatisticsData.thrift_spec = (
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 783259c6f10..6b09ac638ef 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -1895,13 +1895,15 @@ class DoubleColumnStatsData
NUMNULLS = 3
NUMDVS = 4
BITVECTORS = 5
+ HISTOGRAM = 6
FIELDS = {
LOWVALUE => {:type => ::Thrift::Types::DOUBLE, :name => 'lowValue', :optional => true},
HIGHVALUE => {:type => ::Thrift::Types::DOUBLE, :name => 'highValue', :optional => true},
NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
- BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true},
+ HISTOGRAM => {:type => ::Thrift::Types::STRING, :name => 'histogram', :binary => true, :optional => true}
}
def struct_fields; FIELDS; end
@@ -1921,13 +1923,15 @@ class LongColumnStatsData
NUMNULLS = 3
NUMDVS = 4
BITVECTORS = 5
+ HISTOGRAM = 6
FIELDS = {
LOWVALUE => {:type => ::Thrift::Types::I64, :name => 'lowValue', :optional => true},
HIGHVALUE => {:type => ::Thrift::Types::I64, :name => 'highValue', :optional => true},
NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
- BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true},
+ HISTOGRAM => {:type => ::Thrift::Types::STRING, :name => 'histogram', :binary => true, :optional => true}
}
def struct_fields; FIELDS; end
@@ -2020,13 +2024,15 @@ class DecimalColumnStatsData
NUMNULLS = 3
NUMDVS = 4
BITVECTORS = 5
+ HISTOGRAM = 6
FIELDS = {
LOWVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'lowValue', :class => ::Decimal, :optional => true},
HIGHVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'highValue', :class => ::Decimal, :optional => true},
NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
- BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true},
+ HISTOGRAM => {:type => ::Thrift::Types::STRING, :name => 'histogram', :binary => true, :optional => true}
}
def struct_fields; FIELDS; end
@@ -2063,13 +2069,15 @@ class DateColumnStatsData
NUMNULLS = 3
NUMDVS = 4
BITVECTORS = 5
+ HISTOGRAM = 6
FIELDS = {
LOWVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'lowValue', :class => ::Date, :optional => true},
HIGHVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'highValue', :class => ::Date, :optional => true},
NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
- BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true},
+ HISTOGRAM => {:type => ::Thrift::Types::STRING, :name => 'histogram', :binary => true, :optional => true}
}
def struct_fields; FIELDS; end
@@ -2106,13 +2114,15 @@ class TimestampColumnStatsData
NUMNULLS = 3
NUMDVS = 4
BITVECTORS = 5
+ HISTOGRAM = 6
FIELDS = {
LOWVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'lowValue', :class => ::Timestamp, :optional => true},
HIGHVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'highValue', :class => ::Timestamp, :optional => true},
NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
- BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+ BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true},
+ HISTOGRAM => {:type => ::Thrift::Types::STRING, :name => 'histogram', :binary => true, :optional => true}
}
def struct_fields; FIELDS; end
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 460e41b418e..3085fb33cf8 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1319,6 +1319,8 @@ public class MetastoreConf {
"A flag to gather statistics (only basic) automatically during the INSERT OVERWRITE command."),
STATS_FETCH_BITVECTOR("metastore.stats.fetch.bitvector", "hive.stats.fetch.bitvector", false,
"Whether we fetch bitvector when we compute ndv. Users can turn it off if they want to use old schema"),
+ STATS_FETCH_KLL("metastore.stats.fetch.kll", "hive.stats.fetch.kll", false,
+ "Whether we fetch KLL data sketches to enable histogram statistics"),
STATS_NDV_TUNER("metastore.stats.ndv.tuner", "hive.metastore.stats.ndv.tuner", 0.0,
"Provides a tunable parameter between the lower bound and the higher bound of ndv for aggregate ndv across all the partitions. \n" +
"The lower bound is equal to the maximum of ndv of all the partitions. The higher bound is equal to the sum of ndv of all the partitions.\n" +
diff --git a/standalone-metastore/metastore-common/src/main/protobuf/org/apache/hadoop/hive/metastore/hive_metastore.proto b/standalone-metastore/metastore-common/src/main/protobuf/org/apache/hadoop/hive/metastore/hive_metastore.proto
index c34402f46c5..56cf6c9109b 100644
--- a/standalone-metastore/metastore-common/src/main/protobuf/org/apache/hadoop/hive/metastore/hive_metastore.proto
+++ b/standalone-metastore/metastore-common/src/main/protobuf/org/apache/hadoop/hive/metastore/hive_metastore.proto
@@ -365,6 +365,7 @@ message DoubleColumnStatsData {
int64 num_nulls = 3;
int64 num_dvs = 4;
bytes bit_vectors = 5;
+ bytes histogram = 6;
}
message LongColumnStatsData {
@@ -373,6 +374,7 @@ message LongColumnStatsData {
int64 num_nulls = 3;
int64 num_dvs = 4;
bytes bit_vectors = 5;
+ bytes histogram = 6;
}
message StringColumnStatsData {
@@ -401,6 +403,7 @@ message DecimalColumnStatsData {
int64 num_nulls = 3;
int64 num_dvs = 4;
bytes bit_vectors = 5;
+ bytes histogram = 6;
}
message Date {
@@ -413,6 +416,7 @@ message DateColumnStatsData {
int64 num_nulls = 3;
int64 num_dvs = 4;
bytes bit_vectors = 5;
+ bytes histogram = 6;
}
message TimestampColumnStatsData {
@@ -421,6 +425,7 @@ message TimestampColumnStatsData {
int64 num_nulls = 3;
int64 num_dvs = 4;
bytes bit_vectors = 5;
+ bytes histogram = 6;
}
message ColumnStatisticsData {
diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index d75a73ee2cb..179a4960b90 100644
--- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -169,7 +169,7 @@ enum PartitionEventType {
LOAD_DONE = 1,
}
-// Enums for transaction and lock management
+// Enums for transaction and lock management
enum TxnState {
COMMITTED = 1,
ABORTED = 2,
@@ -480,7 +480,8 @@ struct DoubleColumnStatsData {
2: optional double highValue,
3: required i64 numNulls,
4: required i64 numDVs,
-5: optional binary bitVectors
+5: optional binary bitVectors,
+6: optional binary histogram
}
struct LongColumnStatsData {
@@ -488,7 +489,8 @@ struct LongColumnStatsData {
2: optional i64 highValue,
3: required i64 numNulls,
4: required i64 numDVs,
-5: optional binary bitVectors
+5: optional binary bitVectors,
+6: optional binary histogram
}
struct StringColumnStatsData {
@@ -517,7 +519,8 @@ struct DecimalColumnStatsData {
2: optional Decimal highValue,
3: required i64 numNulls,
4: required i64 numDVs,
-5: optional binary bitVectors
+5: optional binary bitVectors,
+6: optional binary histogram
}
struct Date {
@@ -529,7 +532,8 @@ struct DateColumnStatsData {
2: optional Date highValue,
3: required i64 numNulls,
4: required i64 numDVs,
-5: optional binary bitVectors
+5: optional binary bitVectors,
+6: optional binary histogram
}
struct Timestamp {
@@ -541,7 +545,8 @@ struct TimestampColumnStatsData {
2: optional Timestamp highValue,
3: required i64 numNulls,
4: required i64 numDVs,
-5: optional binary bitVectors
+5: optional binary bitVectors,
+6: optional binary histogram
}
union ColumnStatisticsData {
@@ -1703,7 +1708,7 @@ struct DropDatabaseRequest {
7: optional i64 txnId=0,
8: optional bool deleteManagedDir=true
}
-
+
// Request type for cm_recycle
struct CmRecycleRequest {
1: required string dataPath,
@@ -3180,7 +3185,7 @@ const string PARTITION_TRANSFORM_SPEC = "partition_transform_spec",
const string NO_CLEANUP = "no_cleanup",
const string CTAS_LEGACY_CONFIG = "create_table_as_external",
const string DEFAULT_TABLE_TYPE = "defaultTableType",
-
+
// ACID
const string TXN_ID = "txnId",
const string WRITE_ID = "writeId",
diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml
index 1c185fba6f9..5e9e1260390 100644
--- a/standalone-metastore/metastore-server/pom.xml
+++ b/standalone-metastore/metastore-server/pom.xml
@@ -44,6 +44,16 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.datasketches</groupId>
+ <artifactId>datasketches-hive</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/datasketches/kll/KllHistogramEstimator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/histogram/KllHistogramEstimator.java
similarity index 87%
rename from ql/src/java/org/apache/hadoop/hive/ql/udf/datasketches/kll/KllHistogramEstimator.java
rename to standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/histogram/KllHistogramEstimator.java
index 4d0777c3ff9..24be2cde0f8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/datasketches/kll/KllHistogramEstimator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/histogram/KllHistogramEstimator.java
@@ -15,9 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.udf.datasketches.kll;
+package org.apache.hadoop.hive.common.histogram;
import org.apache.datasketches.kll.KllFloatsSketch;
+import org.apache.hadoop.hive.common.histogram.kll.KllUtils;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
@@ -28,6 +29,10 @@ public class KllHistogramEstimator {
private final KllFloatsSketch kll;
+ KllHistogramEstimator() {
+ this.kll = new KllFloatsSketch();
+ }
+
KllHistogramEstimator(int k) {
this.kll = new KllFloatsSketch(k);
}
@@ -68,11 +73,15 @@ public class KllHistogramEstimator {
return KllUtils.lengthFor(model, kll);
}
- public KllFloatsSketch getSketch() {
- return kll;
+ public boolean canMerge(KllHistogramEstimator o) {
+ return o != null && this.kll.getK() == o.kll.getK();
}
public int getK() {
return kll.getK();
}
+
+ public KllFloatsSketch getSketch() {
+ return kll;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/datasketches/kll/KllHistogramEstimatorFactory.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/histogram/KllHistogramEstimatorFactory.java
similarity index 58%
rename from ql/src/java/org/apache/hadoop/hive/ql/udf/datasketches/kll/KllHistogramEstimatorFactory.java
rename to standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/histogram/KllHistogramEstimatorFactory.java
index 72dc7d3b400..075e432b2be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/datasketches/kll/KllHistogramEstimatorFactory.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/histogram/KllHistogramEstimatorFactory.java
@@ -17,7 +17,9 @@
* under the License.
*/
-package org.apache.hadoop.hive.ql.udf.datasketches.kll;
+package org.apache.hadoop.hive.common.histogram;
+
+import org.apache.hadoop.hive.common.histogram.kll.KllUtils;
public class KllHistogramEstimatorFactory {
@@ -25,6 +27,15 @@ public class KllHistogramEstimatorFactory {
throw new AssertionError("Suppress default constructor for non instantiation");
}
+ /**
+ * This function deserializes the serialized KLL histogram estimator from a byte array.
+ * @param buf to deserialize
+ * @return KLL histogram estimator
+ */
+ public static KllHistogramEstimator getKllHistogramEstimator(byte[] buf) {
+ return new KllHistogramEstimator(KllUtils.deserializeKll(buf, 0, buf.length));
+ }
+
/**
* This function deserializes the serialized KLL histogram estimator from a byte array.
* @param buf to deserialize
@@ -36,6 +47,14 @@ public class KllHistogramEstimatorFactory {
return new KllHistogramEstimator(KllUtils.deserializeKll(buf, start, len));
}
+ /**
+ * This method creates an empty histogram estimator with a KLL sketch with k=200.
+ * @return an empty histogram estimator with a KLL sketch with k=200
+ */
+ public static KllHistogramEstimator getEmptyHistogramEstimator() {
+ return new KllHistogramEstimator();
+ }
+
/**
* This method creates an empty histogram estimator with a KLL sketch of a given k parameter.
* @param k the KLL parameter k for initializing the sketch
@@ -44,4 +63,14 @@ public class KllHistogramEstimatorFactory {
public static KllHistogramEstimator getEmptyHistogramEstimator(int k) {
return new KllHistogramEstimator(k);
}
+
+ /**
+ * This method creates an empty KLL histogram estimator, using the k parameter from the given estimator.
+ * @param kllHistogramEstimator the estimator used to build the new one
+ * @return an empty KLL histogram estimator, using the k parameter from the given estimator
+ */
+ public static KllHistogramEstimator getEmptyHistogramEstimator(KllHistogramEstimator kllHistogramEstimator) {
+ return kllHistogramEstimator == null ? getEmptyHistogramEstimator()
+ : getEmptyHistogramEstimator(kllHistogramEstimator.getSketch().getK());
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/datasketches/kll/KllUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/histogram/kll/KllUtils.java
similarity index 91%
rename from ql/src/java/org/apache/hadoop/hive/ql/udf/datasketches/kll/KllUtils.java
rename to standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/histogram/kll/KllUtils.java
index 2d9c08b586d..f683c50c5e6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/datasketches/kll/KllUtils.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/common/histogram/kll/KllUtils.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.udf.datasketches.kll;
+package org.apache.hadoop.hive.common.histogram.kll;
import org.apache.datasketches.kll.KllFloatsSketch;
import org.apache.datasketches.memory.Memory;
@@ -84,6 +84,15 @@ public class KllUtils {
}
}
+ /**
+ * This function deserializes the serialized KLL sketch from a byte array.
+ * @param buf to deserialize
+ * @return KLL sketch
+ */
+ public static KllFloatsSketch deserializeKll(final byte[] buf) {
+ return deserializeKll(buf, 0, buf.length);
+ }
+
/**
* Returns the length of the given KLL sketch according to the given java data model.
* @param model the java data model to compute the length
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdateStat.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdateStat.java
index 648fb0dc4d7..1f5e773bc2c 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdateStat.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdateStat.java
@@ -233,8 +233,8 @@ class DirectSqlUpdateStat {
+ "\"TABLE_NAME\", \"PARTITION_NAME\", \"COLUMN_NAME\", \"COLUMN_TYPE\", \"PART_ID\","
+ " \"LONG_LOW_VALUE\", \"LONG_HIGH_VALUE\", \"DOUBLE_HIGH_VALUE\", \"DOUBLE_LOW_VALUE\","
+ " \"BIG_DECIMAL_LOW_VALUE\", \"BIG_DECIMAL_HIGH_VALUE\", \"NUM_NULLS\", \"NUM_DISTINCTS\", \"BIT_VECTOR\" ,"
- + " \"AVG_COL_LEN\", \"MAX_COL_LEN\", \"NUM_TRUES\", \"NUM_FALSES\", \"LAST_ANALYZED\", \"ENGINE\") values "
- + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ + " \"HISTOGRAM\", \"AVG_COL_LEN\", \"MAX_COL_LEN\", \"NUM_TRUES\", \"NUM_FALSES\", \"LAST_ANALYZED\", \"ENGINE\") values "
+ + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
try {
preparedStatement = dbConn.prepareStatement(insert);
@@ -260,12 +260,13 @@ class DirectSqlUpdateStat {
preparedStatement.setObject(15, mPartitionColumnStatistics.getNumNulls());
preparedStatement.setObject(16, mPartitionColumnStatistics.getNumDVs());
preparedStatement.setObject(17, mPartitionColumnStatistics.getBitVector());
- preparedStatement.setObject(18, mPartitionColumnStatistics.getAvgColLen());
- preparedStatement.setObject(19, mPartitionColumnStatistics.getMaxColLen());
- preparedStatement.setObject(20, mPartitionColumnStatistics.getNumTrues());
- preparedStatement.setObject(21, mPartitionColumnStatistics.getNumFalses());
- preparedStatement.setLong(22, mPartitionColumnStatistics.getLastAnalyzed());
- preparedStatement.setString(23, mPartitionColumnStatistics.getEngine());
+ preparedStatement.setObject(18, mPartitionColumnStatistics.getHistogram());
+ preparedStatement.setObject(19, mPartitionColumnStatistics.getAvgColLen());
+ preparedStatement.setObject(20, mPartitionColumnStatistics.getMaxColLen());
+ preparedStatement.setObject(21, mPartitionColumnStatistics.getNumTrues());
+ preparedStatement.setObject(22, mPartitionColumnStatistics.getNumFalses());
+ preparedStatement.setLong(23, mPartitionColumnStatistics.getLastAnalyzed());
+ preparedStatement.setString(24, mPartitionColumnStatistics.getEngine());
maxCsId++;
numRows++;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 0272ff96986..e9ad23856de 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -1404,12 +1404,12 @@ class MetaStoreDirectSql {
*/
public ColumnStatistics getTableStats(final String catName, final String dbName,
final String tableName, List<String> colNames, String engine,
- boolean enableBitVector) throws MetaException {
+ boolean enableBitVector, boolean enableKll) throws MetaException {
if (colNames == null || colNames.isEmpty()) {
return null;
}
final boolean doTrace = LOG.isDebugEnabled();
- final String queryText0 = "select " + getStatsList(enableBitVector) + " from " + TAB_COL_STATS
+ final String queryText0 = "select " + getStatsList(enableBitVector, enableKll) + " from " + TAB_COL_STATS
+ " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ " and \"ENGINE\" = ? and \"COLUMN_NAME\" in (";
Batchable<String, Object[]> b = new Batchable<String, Object[]>() {
@@ -1453,8 +1453,7 @@ class MetaStoreDirectSql {
}
ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tableName);
csd.setCatName(catName);
- ColumnStatistics result = makeColumnStats(list, csd, 0, engine);
- return result;
+ return makeColumnStats(list, csd, 0, engine);
}
public List<HiveObjectPrivilege> getTableAllColumnGrants(String catName, String dbName,
@@ -1540,7 +1539,7 @@ class MetaStoreDirectSql {
public AggrStats aggrColStatsForPartitions(String catName, String dbName, String tableName,
List<String> partNames, List<String> colNames, String engine,
- boolean useDensityFunctionForNDVEstimation, double ndvTuner, boolean enableBitVector)
+ boolean useDensityFunctionForNDVEstimation, double ndvTuner, boolean enableBitVector, boolean enableKll)
throws MetaException {
if (colNames.isEmpty() || partNames.isEmpty()) {
LOG.debug("Columns is empty or partNames is empty : Short-circuiting stats eval");
@@ -1575,7 +1574,7 @@ class MetaStoreDirectSql {
// Read aggregated stats for one column
colStatsAggrFromDB =
columnStatisticsObjForPartitions(catName, dbName, tableName, partNames, colNamesForDB, engine,
- partsFound, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector);
+ partsFound, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll);
if (!colStatsAggrFromDB.isEmpty()) {
ColumnStatisticsObj colStatsAggr = colStatsAggrFromDB.get(0);
colStatsList.add(colStatsAggr);
@@ -1588,7 +1587,7 @@ class MetaStoreDirectSql {
partsFound = partsFoundForPartitions(catName, dbName, tableName, partNames, colNames, engine);
colStatsList =
columnStatisticsObjForPartitions(catName, dbName, tableName, partNames, colNames, engine, partsFound,
- useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector);
+ useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll);
}
LOG.debug("useDensityFunctionForNDVEstimation = " + useDensityFunctionForNDVEstimation
+ "\npartsFound = " + partsFound + "\nColumnStatisticsObj = "
@@ -1653,7 +1652,7 @@ class MetaStoreDirectSql {
private List<ColumnStatisticsObj> columnStatisticsObjForPartitions(
final String catName, final String dbName, final String tableName, final List<String> partNames,
List<String> colNames, String engine, long partsFound, final boolean useDensityFunctionForNDVEstimation,
- final double ndvTuner, final boolean enableBitVector) throws MetaException {
+ final double ndvTuner, final boolean enableBitVector, boolean enableKll) throws MetaException {
final boolean areAllPartsFound = (partsFound == partNames.size());
return Batchable.runBatched(batchSize, colNames, new Batchable<String, ColumnStatisticsObj>() {
@Override
@@ -1662,7 +1661,8 @@ class MetaStoreDirectSql {
@Override
public List<ColumnStatisticsObj> run(List<String> inputPartNames) throws MetaException {
return columnStatisticsObjForPartitionsBatch(catName, dbName, tableName, inputPartNames,
- inputColNames, engine, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector);
+ inputColNames, engine, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner,
+ enableBitVector, enableKll);
}
});
}
@@ -1670,8 +1670,8 @@ class MetaStoreDirectSql {
}
public List<ColStatsObjWithSourceInfo> getColStatsForAllTablePartitions(String catName, String dbName,
- boolean enableBitVector) throws MetaException {
- String queryText = "select \"TABLE_NAME\", \"PARTITION_NAME\", " + getStatsList(enableBitVector)
+ boolean enableBitVector, boolean enableKll) throws MetaException {
+ String queryText = "select \"TABLE_NAME\", \"PARTITION_NAME\", " + getStatsList(enableBitVector, enableKll)
+ " from " + " " + PART_COL_STATS + " where \"DB_NAME\" = ? and \"CAT_NAME\" = ?";
long start = 0;
long end = 0;
@@ -1701,11 +1701,12 @@ class MetaStoreDirectSql {
/** Should be called with the list short enough to not trip up Oracle/etc. */
private List<ColumnStatisticsObj> columnStatisticsObjForPartitionsBatch(String catName, String dbName,
String tableName, List<String> partNames, List<String> colNames, String engine,
- boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner, boolean enableBitVector)
+ boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner,
+ boolean enableBitVector, boolean enableKll)
throws MetaException {
- if (enableBitVector) {
+ if (enableBitVector || enableKll) {
return aggrStatsUseJava(catName, dbName, tableName, partNames, colNames, engine, areAllPartsFound,
- useDensityFunctionForNDVEstimation, ndvTuner);
+ useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll);
} else {
return aggrStatsUseDB(catName, dbName, tableName, partNames, colNames, engine, areAllPartsFound,
useDensityFunctionForNDVEstimation, ndvTuner);
@@ -1714,10 +1715,11 @@ class MetaStoreDirectSql {
private List<ColumnStatisticsObj> aggrStatsUseJava(String catName, String dbName, String tableName,
List<String> partNames, List<String> colNames, String engine, boolean areAllPartsFound,
- boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException {
+ boolean useDensityFunctionForNDVEstimation, double ndvTuner, boolean enableBitVector,
+ boolean enableKll) throws MetaException {
// 1. get all the stats for colNames in partNames;
List<ColumnStatistics> partStats =
- getPartitionStats(catName, dbName, tableName, partNames, colNames, engine, true);
+ getPartitionStats(catName, dbName, tableName, partNames, colNames, engine, enableBitVector, enableKll);
// 2. use util function to aggr stats
return MetaStoreServerUtils.aggrPartitionStats(partStats, catName, dbName, tableName, partNames, colNames,
areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner);
@@ -2008,9 +2010,9 @@ class MetaStoreDirectSql {
ColumnStatisticsObj cso = new ColumnStatisticsObj((String)row[i++], (String)row[i++], data);
Object llow = row[i++], lhigh = row[i++], dlow = row[i++], dhigh = row[i++],
declow = row[i++], dechigh = row[i++], nulls = row[i++], dist = row[i++], bitVector = row[i++],
- avglen = row[i++], maxlen = row[i++], trues = row[i++], falses = row[i++];
+ histogram = row[i++], avglen = row[i++], maxlen = row[i++], trues = row[i++], falses = row[i];
StatObjectConverter.fillColumnStatisticsData(cso.getColType(), data,
- llow, lhigh, dlow, dhigh, declow, dechigh, nulls, dist, bitVector, avglen, maxlen, trues, falses);
+ llow, lhigh, dlow, dhigh, declow, dechigh, nulls, dist, bitVector, histogram, avglen, maxlen, trues, falses);
return cso;
}
@@ -2018,7 +2020,10 @@ class MetaStoreDirectSql {
boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException {
ColumnStatisticsData data = new ColumnStatisticsData();
ColumnStatisticsObj cso = new ColumnStatisticsObj((String) row[i++], (String) row[i++], data);
- Object llow = row[i++], lhigh = row[i++], dlow = row[i++], dhigh = row[i++], declow = row[i++], dechigh = row[i++], nulls = row[i++], dist = row[i++], avglen = row[i++], maxlen = row[i++], trues = row[i++], falses = row[i++], avgLong = row[i++], avgDouble = row[i++], avgDecimal = row[i++], sumDist = row[i++];
+ Object llow = row[i++], lhigh = row[i++], dlow = row[i++], dhigh = row[i++], declow = row[i++],
+ dechigh = row[i++], nulls = row[i++], dist = row[i++], avglen = row[i++], maxlen = row[i++],
+ trues = row[i++], falses = row[i++], avgLong = row[i++], avgDouble = row[i++],
+ avgDecimal = row[i++], sumDist = row[i++];
StatObjectConverter.fillColumnStatisticsData(cso.getColType(), data, llow, lhigh, dlow, dhigh,
declow, dechigh, nulls, dist, avglen, maxlen, trues, falses, avgLong, avgDouble,
avgDecimal, sumDist, useDensityFunctionForNDVEstimation, ndvTuner);
@@ -2045,12 +2050,12 @@ class MetaStoreDirectSql {
public List<ColumnStatistics> getPartitionStats(
final String catName, final String dbName, final String tableName, final List<String> partNames,
- List<String> colNames, String engine, boolean enableBitVector) throws MetaException {
+ List<String> colNames, String engine, boolean enableBitVector, boolean enableKll) throws MetaException {
if (colNames.isEmpty() || partNames.isEmpty()) {
return Collections.emptyList();
}
final boolean doTrace = LOG.isDebugEnabled();
- final String queryText0 = "select \"PARTITION_NAME\", " + getStatsList(enableBitVector) + " from "
+ final String queryText0 = "select \"PARTITION_NAME\", " + getStatsList(enableBitVector, enableKll) + " from "
+ " " + PART_COL_STATS + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and " +
"\"COLUMN_NAME\""
+ " in (%1$s) AND \"PARTITION_NAME\" in (%2$s) "
@@ -2115,12 +2120,13 @@ class MetaStoreDirectSql {
}
/** The common query part for table and partition stats */
- private final String getStatsList(boolean enableBitVector) {
+ private String getStatsList(boolean enableBitVector, boolean enableKll) {
return "\"COLUMN_NAME\", \"COLUMN_TYPE\", \"LONG_LOW_VALUE\", \"LONG_HIGH_VALUE\", "
+ "\"DOUBLE_LOW_VALUE\", \"DOUBLE_HIGH_VALUE\", \"BIG_DECIMAL_LOW_VALUE\", "
+ "\"BIG_DECIMAL_HIGH_VALUE\", \"NUM_NULLS\", \"NUM_DISTINCTS\", "
- + (enableBitVector ? "\"BIT_VECTOR\", " : "\'\', ") + "\"AVG_COL_LEN\", "
- + "\"MAX_COL_LEN\", \"NUM_TRUES\", \"NUM_FALSES\", \"LAST_ANALYZED\" ";
+ + (enableBitVector ? "\"BIT_VECTOR\", " : "\'\', ")
+ + (enableKll ? "\"HISTOGRAM\", " : "\'\', ")
+ + "\"AVG_COL_LEN\", \"MAX_COL_LEN\", \"NUM_TRUES\", \"NUM_FALSES\", \"LAST_ANALYZED\" ";
}
private ColumnStatistics makeColumnStats(
@@ -2131,7 +2137,7 @@ class MetaStoreDirectSql {
for (Object[] row : list) {
// LastAnalyzed is stored per column but thrift has it per several;
// get the lowest for now as nobody actually uses this field.
- Object laObj = row[offset + 15];
+ Object laObj = row[offset + 16]; // 16 is the offset of "last analyzed" field
if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > MetastoreDirectSqlUtils
.extractSqlLong(laObj))) {
csd.setLastAnalyzed(MetastoreDirectSqlUtils.extractSqlLong(laObj));
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 9fe936d26ed..06f60478875 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -10079,11 +10079,12 @@ public class ObjectStore implements RawStore, Configurable {
String catName, String dbName, String tableName, final List<String> colNames, String engine,
boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException {
final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_KLL);
return new GetStatHelper(normalizeIdentifier(catName), normalizeIdentifier(dbName),
normalizeIdentifier(tableName), allowSql, allowJdo, null) {
@Override
protected ColumnStatistics getSqlResult(GetHelper<ColumnStatistics> ctx) throws MetaException {
- return directSql.getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector);
+ return directSql.getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector, enableKll);
}
@Override
@@ -10102,7 +10103,7 @@ public class ObjectStore implements RawStore, Configurable {
if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) {
desc.setLastAnalyzed(mStat.getLastAnalyzed());
}
- statObjs.add(StatObjectConverter.getTableColumnStatisticsObj(mStat, enableBitVector));
+ statObjs.add(StatObjectConverter.getTableColumnStatisticsObj(mStat, enableBitVector, enableKll));
Deadline.checkTimeout();
}
ColumnStatistics colStat = new ColumnStatistics(desc, statObjs);
@@ -10202,11 +10203,13 @@ public class ObjectStore implements RawStore, Configurable {
String catName, String dbName, String tableName, final List<String> partNames, final List<String> colNames,
String engine, boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException {
final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_KLL);
return new GetListHelper<ColumnStatistics>(catName, dbName, tableName, allowSql, allowJdo) {
@Override
protected List<ColumnStatistics> getSqlResult(
GetHelper<List<ColumnStatistics>> ctx) throws MetaException {
- return directSql.getPartitionStats(catName, dbName, tblName, partNames, colNames, engine, enableBitVector);
+ return directSql.getPartitionStats(
+ catName, dbName, tblName, partNames, colNames, engine, enableBitVector, enableKll);
}
@Override
protected List<ColumnStatistics> getJdoResult(GetHelper<List<ColumnStatistics>> ctx)
@@ -10233,7 +10236,7 @@ public class ObjectStore implements RawStore, Configurable {
csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj);
curList = new ArrayList<>(colNames.size());
}
- curList.add(StatObjectConverter.getPartitionColumnStatisticsObj(mStatsObj, enableBitVector));
+ curList.add(StatObjectConverter.getPartitionColumnStatisticsObj(mStatsObj, enableBitVector, enableKll));
lastPartName = partName;
Deadline.checkTimeout();
}
@@ -10290,12 +10293,13 @@ public class ObjectStore implements RawStore, Configurable {
ConfVars.STATS_NDV_DENSITY_FUNCTION);
final double ndvTuner = MetastoreConf.getDoubleVar(getConf(), ConfVars.STATS_NDV_TUNER);
final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_KLL);
return new GetHelper<AggrStats>(catName, dbName, tblName, true, false) {
@Override
protected AggrStats getSqlResult(GetHelper<AggrStats> ctx)
throws MetaException {
return directSql.aggrColStatsForPartitions(catName, dbName, tblName, partNames,
- colNames, engine, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector);
+ colNames, engine, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll);
}
@Override
protected AggrStats getJdoResult(GetHelper<AggrStats> ctx)
@@ -10315,14 +10319,14 @@ public class ObjectStore implements RawStore, Configurable {
@Override
public List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String catName, String dbName)
throws MetaException, NoSuchObjectException {
- final boolean enableBitVector =
- MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR);
+ final boolean enableKll = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_KLL);
return new GetHelper<List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>>(
catName, dbName, null, true, false) {
@Override
protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getSqlResult(
GetHelper<List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>> ctx) throws MetaException {
- return directSql.getColStatsForAllTablePartitions(catName, dbName, enableBitVector);
+ return directSql.getColStatsForAllTablePartitions(catName, dbName, enableBitVector, enableKll);
}
@Override
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
index 51eddd76d65..5848abd2064 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
@@ -88,6 +88,7 @@ public class StatObjectConverter {
longStats.isSetNumNulls() ? longStats.getNumNulls() : null,
longStats.isSetNumDVs() ? longStats.getNumDVs() : null,
longStats.isSetBitVectors() ? longStats.getBitVectors() : null,
+ longStats.isSetHistogram() ? longStats.getHistogram() : null,
longStats.isSetLowValue() ? longStats.getLowValue() : null,
longStats.isSetHighValue() ? longStats.getHighValue() : null);
} else if (statsObj.getStatsData().isSetDoubleStats()) {
@@ -96,6 +97,7 @@ public class StatObjectConverter {
doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null,
doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null,
doubleStats.isSetBitVectors() ? doubleStats.getBitVectors() : null,
+ doubleStats.isSetHistogram() ? doubleStats.getHistogram() : null,
doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null,
doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null);
} else if (statsObj.getStatsData().isSetDecimalStats()) {
@@ -106,6 +108,7 @@ public class StatObjectConverter {
decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null,
decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null,
decimalStats.isSetBitVectors() ? decimalStats.getBitVectors() : null,
+ decimalStats.isSetHistogram() ? decimalStats.getHistogram() : null,
low, high);
} else if (statsObj.getStatsData().isSetStringStats()) {
StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
@@ -127,6 +130,7 @@ public class StatObjectConverter {
dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null,
dateStats.isSetNumDVs() ? dateStats.getNumDVs() : null,
dateStats.isSetBitVectors() ? dateStats.getBitVectors() : null,
+ dateStats.isSetHistogram() ? dateStats.getHistogram() : null,
dateStats.isSetLowValue() ? dateStats.getLowValue().getDaysSinceEpoch() : null,
dateStats.isSetHighValue() ? dateStats.getHighValue().getDaysSinceEpoch() : null);
} else if (statsObj.getStatsData().isSetTimestampStats()) {
@@ -135,6 +139,7 @@ public class StatObjectConverter {
timestampStats.isSetNumNulls() ? timestampStats.getNumNulls() : null,
timestampStats.isSetNumDVs() ? timestampStats.getNumDVs() : null,
timestampStats.isSetBitVectors() ? timestampStats.getBitVectors() : null,
+ timestampStats.isSetHistogram() ? timestampStats.getHistogram() : null,
timestampStats.isSetLowValue() ? timestampStats.getLowValue().getSecondsSinceEpoch() : null,
timestampStats.isSetHighValue() ? timestampStats.getHighValue().getSecondsSinceEpoch() : null);
}
@@ -174,6 +179,9 @@ public class StatObjectConverter {
if (mStatsObj.getBitVector() != null) {
oldStatsObj.setBitVector(mStatsObj.getBitVector());
}
+ if (mStatsObj.getHistogram() != null) {
+ oldStatsObj.setHistogram(mStatsObj.getHistogram());
+ }
if (mStatsObj.getNumFalses() != null) {
oldStatsObj.setNumFalses(mStatsObj.getNumFalses());
}
@@ -220,6 +228,9 @@ public class StatObjectConverter {
if (mStatsObj.getBitVector() != null) {
oldStatsObj.setBitVector(mStatsObj.getBitVector());
}
+ if (mStatsObj.getHistogram() != null) {
+ oldStatsObj.setHistogram(mStatsObj.getHistogram());
+ }
if (mStatsObj.getNumFalses() != null) {
oldStatsObj.setNumFalses(mStatsObj.getNumFalses());
}
@@ -265,6 +276,9 @@ public class StatObjectConverter {
if (mStatsObj.getBitVector() != null) {
setStmt.append("\"BIT_VECTOR\" = ? ,");
}
+ if (mStatsObj.getHistogram() != null) {
+ setStmt.append("\"HISTOGRAM\" = ? ,");
+ }
if (mStatsObj.getNumFalses() != null) {
setStmt.append("\"NUM_FALSES\" = ? ,");
}
@@ -312,6 +326,9 @@ public class StatObjectConverter {
if (mStatsObj.getBitVector() != null) {
pst.setObject(colIdx++, mStatsObj.getBitVector());
}
+ if (mStatsObj.getHistogram() != null) {
+ pst.setObject(colIdx++, mStatsObj.getHistogram());
+ }
if (mStatsObj.getNumFalses() != null) {
pst.setObject(colIdx++, mStatsObj.getNumFalses());
}
@@ -325,7 +342,7 @@ public class StatObjectConverter {
}
public static ColumnStatisticsObj getTableColumnStatisticsObj(
- MTableColumnStatistics mStatsObj, boolean enableBitVector) {
+ MTableColumnStatistics mStatsObj, boolean enableBitVector, boolean enableKll) {
ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
statsObj.setColType(mStatsObj.getColType());
statsObj.setColName(mStatsObj.getColName());
@@ -367,6 +384,7 @@ public class StatObjectConverter {
}
longStats.setNumDVs(mStatsObj.getNumDVs());
longStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ longStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setLongStats(longStats);
} else if (colType.equals("double") || colType.equals("float")) {
DoubleColumnStatsDataInspector doubleStats = new DoubleColumnStatsDataInspector();
@@ -381,6 +399,7 @@ public class StatObjectConverter {
}
doubleStats.setNumDVs(mStatsObj.getNumDVs());
doubleStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ doubleStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setDoubleStats(doubleStats);
} else if (colType.startsWith("decimal")) {
DecimalColumnStatsDataInspector decimalStats = new DecimalColumnStatsDataInspector();
@@ -395,6 +414,7 @@ public class StatObjectConverter {
}
decimalStats.setNumDVs(mStatsObj.getNumDVs());
decimalStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ decimalStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setDecimalStats(decimalStats);
} else if (colType.equals("date")) {
DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector();
@@ -409,6 +429,7 @@ public class StatObjectConverter {
}
dateStats.setNumDVs(mStatsObj.getNumDVs());
dateStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ dateStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setDateStats(dateStats);
} else if (colType.equals("timestamp")) {
TimestampColumnStatsDataInspector timestampStats = new TimestampColumnStatsDataInspector();
@@ -423,6 +444,7 @@ public class StatObjectConverter {
}
timestampStats.setNumDVs(mStatsObj.getNumDVs());
timestampStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ timestampStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setTimestampStats(timestampStats);
}
statsObj.setStatsData(colStatsData);
@@ -474,6 +496,7 @@ public class StatObjectConverter {
longStats.isSetNumNulls() ? longStats.getNumNulls() : null,
longStats.isSetNumDVs() ? longStats.getNumDVs() : null,
longStats.isSetBitVectors() ? longStats.getBitVectors() : null,
+ longStats.isSetHistogram() ? longStats.getHistogram() : null,
longStats.isSetLowValue() ? longStats.getLowValue() : null,
longStats.isSetHighValue() ? longStats.getHighValue() : null);
} else if (statsObj.getStatsData().isSetDoubleStats()) {
@@ -482,6 +505,7 @@ public class StatObjectConverter {
doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null,
doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null,
doubleStats.isSetBitVectors() ? doubleStats.getBitVectors() : null,
+ doubleStats.isSetHistogram() ? doubleStats.getHistogram() : null,
doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null,
doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null);
} else if (statsObj.getStatsData().isSetDecimalStats()) {
@@ -492,6 +516,7 @@ public class StatObjectConverter {
decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null,
decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null,
decimalStats.isSetBitVectors() ? decimalStats.getBitVectors() : null,
+ decimalStats.isSetHistogram() ? decimalStats.getHistogram() : null,
low, high);
} else if (statsObj.getStatsData().isSetStringStats()) {
StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
@@ -513,6 +538,7 @@ public class StatObjectConverter {
dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null,
dateStats.isSetNumDVs() ? dateStats.getNumDVs() : null,
dateStats.isSetBitVectors() ? dateStats.getBitVectors() : null,
+ dateStats.isSetHistogram() ? dateStats.getHistogram() : null,
dateStats.isSetLowValue() ? dateStats.getLowValue().getDaysSinceEpoch() : null,
dateStats.isSetHighValue() ? dateStats.getHighValue().getDaysSinceEpoch() : null);
} else if (statsObj.getStatsData().isSetTimestampStats()) {
@@ -521,6 +547,7 @@ public class StatObjectConverter {
timestampStats.isSetNumNulls() ? timestampStats.getNumNulls() : null,
timestampStats.isSetNumDVs() ? timestampStats.getNumDVs() : null,
timestampStats.isSetBitVectors() ? timestampStats.getBitVectors() : null,
+ timestampStats.isSetHistogram() ? timestampStats.getHistogram() : null,
timestampStats.isSetLowValue() ? timestampStats.getLowValue().getSecondsSinceEpoch() : null,
timestampStats.isSetHighValue() ? timestampStats.getHighValue().getSecondsSinceEpoch() : null);
}
@@ -529,7 +556,7 @@ public class StatObjectConverter {
}
public static ColumnStatisticsObj getPartitionColumnStatisticsObj(
- MPartitionColumnStatistics mStatsObj, boolean enableBitVector) {
+ MPartitionColumnStatistics mStatsObj, boolean enableBitVector, boolean enableKll) {
ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
statsObj.setColType(mStatsObj.getColType());
statsObj.setColName(mStatsObj.getColName());
@@ -569,6 +596,7 @@ public class StatObjectConverter {
}
longStats.setNumDVs(mStatsObj.getNumDVs());
longStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ longStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setLongStats(longStats);
} else if (colType.equals("double") || colType.equals("float")) {
DoubleColumnStatsDataInspector doubleStats = new DoubleColumnStatsDataInspector();
@@ -581,6 +609,7 @@ public class StatObjectConverter {
}
doubleStats.setNumDVs(mStatsObj.getNumDVs());
doubleStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ doubleStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setDoubleStats(doubleStats);
} else if (colType.startsWith("decimal")) {
DecimalColumnStatsDataInspector decimalStats = new DecimalColumnStatsDataInspector();
@@ -593,6 +622,7 @@ public class StatObjectConverter {
}
decimalStats.setNumDVs(mStatsObj.getNumDVs());
decimalStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ decimalStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setDecimalStats(decimalStats);
} else if (colType.equals("date")) {
DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector();
@@ -607,6 +637,7 @@ public class StatObjectConverter {
}
dateStats.setNumDVs(mStatsObj.getNumDVs());
dateStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ dateStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setDateStats(dateStats);
} else if (colType.equals("timestamp")) {
TimestampColumnStatsDataInspector timestampStats = new TimestampColumnStatsDataInspector();
@@ -621,6 +652,7 @@ public class StatObjectConverter {
}
timestampStats.setNumDVs(mStatsObj.getNumDVs());
timestampStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector());
+ timestampStats.setHistogram((mStatsObj.getHistogram()==null||!enableKll)? null : mStatsObj.getHistogram());
colStatsData.setTimestampStats(timestampStats);
}
statsObj.setStatsData(colStatsData);
@@ -649,10 +681,21 @@ public class StatObjectConverter {
return bytes;
}
+ public static byte[] getHistogram(byte[] bytes) {
+ // workaround for DN bug in persisting nulls in pg bytea column
+ // instead set empty bit vector with header.
+ // https://issues.apache.org/jira/browse/HIVE-17836
+ if (bytes != null && bytes.length == 0) {
+ return null;
+ }
+ return bytes;
+ }
+
// JAVA
public static void fillColumnStatisticsData(String colType, ColumnStatisticsData data,
Object llow, Object lhigh, Object dlow, Object dhigh, Object declow, Object dechigh,
- Object nulls, Object dist, Object bitVector, Object avglen, Object maxlen, Object trues, Object falses) throws MetaException {
+ Object nulls, Object dist, Object bitVector, Object histogram,
+ Object avglen, Object maxlen, Object trues, Object falses) throws MetaException {
colType = colType.toLowerCase();
if (colType.equals("boolean")) {
BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
@@ -687,6 +730,7 @@ public class StatObjectConverter {
}
longStats.setNumDVs(MetastoreDirectSqlUtils.extractSqlLong(dist));
longStats.setBitVectors(getBitVector(MetastoreDirectSqlUtils.extractSqlBlob(bitVector)));
+ longStats.setHistogram(getHistogram(MetastoreDirectSqlUtils.extractSqlBlob(histogram)));
data.setLongStats(longStats);
} else if (colType.equals("double") || colType.equals("float")) {
DoubleColumnStatsDataInspector doubleStats = new DoubleColumnStatsDataInspector();
@@ -699,6 +743,7 @@ public class StatObjectConverter {
}
doubleStats.setNumDVs(MetastoreDirectSqlUtils.extractSqlLong(dist));
doubleStats.setBitVectors(getBitVector(MetastoreDirectSqlUtils.extractSqlBlob(bitVector)));
+ doubleStats.setHistogram(getHistogram(MetastoreDirectSqlUtils.extractSqlBlob(histogram)));
data.setDoubleStats(doubleStats);
} else if (colType.startsWith("decimal")) {
DecimalColumnStatsDataInspector decimalStats = new DecimalColumnStatsDataInspector();
@@ -711,6 +756,7 @@ public class StatObjectConverter {
}
decimalStats.setNumDVs(MetastoreDirectSqlUtils.extractSqlLong(dist));
decimalStats.setBitVectors(getBitVector(MetastoreDirectSqlUtils.extractSqlBlob(bitVector)));
+ decimalStats.setHistogram(getHistogram(MetastoreDirectSqlUtils.extractSqlBlob(histogram)));
data.setDecimalStats(decimalStats);
} else if (colType.equals("date")) {
DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector();
@@ -723,6 +769,7 @@ public class StatObjectConverter {
}
dateStats.setNumDVs(MetastoreDirectSqlUtils.extractSqlLong(dist));
dateStats.setBitVectors(getBitVector(MetastoreDirectSqlUtils.extractSqlBlob(bitVector)));
+ dateStats.setHistogram(getHistogram(MetastoreDirectSqlUtils.extractSqlBlob(histogram)));
data.setDateStats(dateStats);
} else if (colType.equals("timestamp")) {
TimestampColumnStatsDataInspector timestampStats = new TimestampColumnStatsDataInspector();
@@ -735,6 +782,7 @@ public class StatObjectConverter {
}
timestampStats.setNumDVs(MetastoreDirectSqlUtils.extractSqlLong(dist));
timestampStats.setBitVectors(getBitVector(MetastoreDirectSqlUtils.extractSqlBlob(bitVector)));
+ timestampStats.setHistogram(getHistogram(MetastoreDirectSqlUtils.extractSqlBlob(histogram)));
data.setTimestampStats(timestampStats);
}
}
@@ -988,6 +1036,9 @@ public class StatObjectConverter {
if (newLongStatsData.isSetBitVectors()) {
oldLongStatsData.setBitVectors(newLongStatsData.getBitVectors());
}
+ if (newLongStatsData.isSetHistogram()) {
+ newLongStatsData.setHistogram(newLongStatsData.getHistogram());
+ }
break;
}
case DOUBLE_STATS: {
@@ -1008,6 +1059,9 @@ public class StatObjectConverter {
if (newDoubleStatsData.isSetBitVectors()) {
oldDoubleStatsData.setBitVectors(newDoubleStatsData.getBitVectors());
}
+ if (newDoubleStatsData.isSetHistogram()) {
+ oldDoubleStatsData.setHistogram(newDoubleStatsData.getHistogram());
+ }
break;
}
case STRING_STATS: {
@@ -1064,6 +1118,9 @@ public class StatObjectConverter {
if (newDecimalStatsData.isSetBitVectors()) {
oldDecimalStatsData.setBitVectors(newDecimalStatsData.getBitVectors());
}
+ if (newDecimalStatsData.isSetHistogram()) {
+ oldDecimalStatsData.setHistogram(newDecimalStatsData.getHistogram());
+ }
break;
}
case DATE_STATS: {
@@ -1084,6 +1141,9 @@ public class StatObjectConverter {
if (newDateStatsData.isSetBitVectors()) {
oldDateStatsData.setBitVectors(newDateStatsData.getBitVectors());
}
+ if (newDateStatsData.isSetHistogram()) {
+ oldDateStatsData.setHistogram(newDateStatsData.getHistogram());
+ }
break;
}
case TIMESTAMP_STATS: {
@@ -1104,6 +1164,9 @@ public class StatObjectConverter {
if (newTimestampStatsData.isSetBitVectors()) {
oldTimestampStatsData.setBitVectors(newTimestampStatsData.getBitVectors());
}
+ if (newTimestampStatsData.isSetHistogram()) {
+ oldTimestampStatsData.setHistogram(newTimestampStatsData.getHistogram());
+ }
break;
}
default:
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
index 552c91835f7..0d645ac7ab0 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
@@ -35,8 +35,8 @@ public class BinaryColumnStatsAggregator extends ColumnStatsAggregator {
checkStatisticsList(colStatsWithSourceInfo);
ColumnStatisticsObj statsObj = null;
- String colType = null;
- String colName = null;
+ String colType;
+ String colName;
BinaryColumnStatsData aggregateData = null;
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
@@ -55,9 +55,13 @@ public class BinaryColumnStatsAggregator extends ColumnStatsAggregator {
aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
}
}
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ ColumnStatisticsData columnStatisticsData = initColumnStatisticsData();
columnStatisticsData.setBinaryStats(aggregateData);
statsObj.setStatsData(columnStatisticsData);
return statsObj;
}
+
+ @Override protected ColumnStatisticsData initColumnStatisticsData() {
+ return new ColumnStatisticsData();
+ }
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
index 9babeea8510..8d58a3b4dac 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
@@ -35,8 +35,8 @@ public class BooleanColumnStatsAggregator extends ColumnStatsAggregator {
checkStatisticsList(colStatsWithSourceInfo);
ColumnStatisticsObj statsObj = null;
- String colType = null;
- String colName = null;
+ String colType;
+ String colName;
BooleanColumnStatsData aggregateData = null;
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
@@ -55,10 +55,13 @@ public class BooleanColumnStatsAggregator extends ColumnStatsAggregator {
aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
}
}
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ ColumnStatisticsData columnStatisticsData = initColumnStatisticsData();
columnStatisticsData.setBooleanStats(aggregateData);
statsObj.setStatsData(columnStatisticsData);
return statsObj;
}
+ @Override protected ColumnStatisticsData initColumnStatisticsData() {
+ return new ColumnStatisticsData();
+ }
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
index 144e71c69ec..adfa6e4f417 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
@@ -19,12 +19,23 @@
package org.apache.hadoop.hive.metastore.columnstats.aggr;
+import java.util.ArrayList;
+import java.util.BitSet;
import java.util.List;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo;
+import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.dateInspectorFromStats;
+import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.decimalInspectorFromStats;
+import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.doubleInspectorFromStats;
+import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.longInspectorFromStats;
+import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.timestampInspectorFromStats;
+
public abstract class ColumnStatsAggregator {
public boolean useDensityFunctionForNDVEstimation;
/**
@@ -51,4 +62,76 @@ public abstract class ColumnStatsAggregator {
throw new IllegalArgumentException("Column statistics list must not be empty when aggregating");
}
}
+
+ protected abstract ColumnStatisticsData initColumnStatisticsData();
+
+ protected KllHistogramEstimator mergeHistograms(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo) {
+ // invariant: no two elements of this list are merge-compatible
+ final List<KllHistogramEstimator> mergedHistogramEstimators = new ArrayList<>();
+ final BitSet mergedStats = new BitSet(colStatsWithSourceInfo.size());
+ int currIndex = 0;
+
+ while (mergedStats.cardinality() != colStatsWithSourceInfo.size()) {
+ currIndex = mergedStats.nextClearBit(currIndex);
+ final ColumnStatisticsObj currColStatsObj = colStatsWithSourceInfo.get(currIndex).getColStatsObj();
+ final KllHistogramEstimator statsEstimator = getHistogramFromStats(currColStatsObj);
+ if (statsEstimator == null) {
+ mergedStats.set(currIndex); // so we can move past it
+ continue;
+ }
+ final KllHistogramEstimator currHistogram = KllHistogramEstimatorFactory.getEmptyHistogramEstimator(statsEstimator);
+ // we need to create a new estimator to not alter the one from the stats object via merging
+ currHistogram.mergeEstimators(statsEstimator);
+
+ // check if the histogram can be merged an existing element in the final list
+ for (KllHistogramEstimator candidateHistogram : mergedHistogramEstimators) {
+ if (candidateHistogram.canMerge(currHistogram)) {
+ candidateHistogram.mergeEstimators(currHistogram);
+ mergedStats.set(currIndex);
+ break;
+ }
+ }
+
+ // if it has not been merged, then store it in the final list
+ if (!mergedStats.get(currIndex)) {
+ mergedHistogramEstimators.add(currHistogram);
+ mergedStats.set(currIndex);
+ }
+ }
+
+ // find the histogram with largest N
+ long biggestN = -1;
+ KllHistogramEstimator largestHistogramEstimator = null;
+ for (KllHistogramEstimator hist : mergedHistogramEstimators) {
+ if (hist.getSketch().getN() > biggestN) {
+ biggestN = hist.getSketch().getN();
+ largestHistogramEstimator = hist;
+ }
+ }
+
+ // set the histogram with largest N
+ return largestHistogramEstimator;
+ }
+
+ private KllHistogramEstimator getHistogramFromStats(ColumnStatisticsObj currColStatsObj) {
+ ColumnStatisticsData columnStatisticsData = currColStatsObj.getStatsData();
+
+ if (columnStatisticsData.isSetDateStats()) {
+ return dateInspectorFromStats(currColStatsObj).getHistogramEstimator();
+ }
+ if (columnStatisticsData.isSetDecimalStats()) {
+ return decimalInspectorFromStats(currColStatsObj).getHistogramEstimator();
+ }
+ if (columnStatisticsData.isSetDoubleStats()) {
+ return doubleInspectorFromStats(currColStatsObj).getHistogramEstimator();
+ }
+ if (columnStatisticsData.isSetLongStats()) {
+ return longInspectorFromStats(currColStatsObj).getHistogramEstimator();
+ }
+ if (columnStatisticsData.isSetTimestampStats()) {
+ return timestampInspectorFromStats(currColStatsObj).getHistogramEstimator();
+ }
+
+ throw new IllegalArgumentException(currColStatsObj.getColType() + " is not supported for merging column stats histograms");
+ }
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java
index a0dcbe9d6a7..9318a05596c 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java
@@ -19,13 +19,13 @@
package org.apache.hadoop.hive.metastore.columnstats.aggr;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
@@ -52,12 +52,13 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
checkStatisticsList(colStatsWithSourceInfo);
ColumnStatisticsObj statsObj = null;
- String colType = null;
+ String colType;
String colName = null;
// check if all the ColumnStatisticsObjs contain stats and all the ndv are
// bitvectors
boolean doAllPartitionContainStats = partNames.size() == colStatsWithSourceInfo.size();
NumDistinctValueEstimator ndvEstimator = null;
+ boolean areAllNDVEstimatorsMergeable = true;
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
if (statsObj == null) {
@@ -67,37 +68,36 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
cso.getStatsData().getSetField());
LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, doAllPartitionContainStats);
}
- DateColumnStatsDataInspector dateColumnStats = dateInspectorFromStats(cso);
+ DateColumnStatsDataInspector columnStatsData = dateInspectorFromStats(cso);
- if (dateColumnStats.getNdvEstimator() == null) {
- ndvEstimator = null;
+ // check if we can merge NDV estimators
+ if (columnStatsData.getNdvEstimator() == null) {
+ areAllNDVEstimatorsMergeable = false;
break;
} else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = dateColumnStats.getNdvEstimator();
+ NumDistinctValueEstimator estimator = columnStatsData.getNdvEstimator();
if (ndvEstimator == null) {
ndvEstimator = estimator;
} else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
+ if (!ndvEstimator.canMerge(estimator)) {
+ areAllNDVEstimatorsMergeable = false;
break;
}
}
}
}
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
+ ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
}
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ LOG.debug("all of the bit vectors can merge for {} is {}", colName, areAllNDVEstimatorsMergeable);
+
+ ColumnStatisticsData columnStatisticsData = initColumnStatisticsData();
if (doAllPartitionContainStats || colStatsWithSourceInfo.size() < 2) {
DateColumnStatsDataInspector aggregateData = null;
long lowerBound = 0;
long higherBound = 0;
double densityAvgSum = 0.0;
+ DateColumnStatsMerger merger = new DateColumnStatsMerger();
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
DateColumnStatsDataInspector newData = dateInspectorFromStats(cso);
@@ -106,13 +106,12 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
if (newData.isSetLowValue() && newData.isSetHighValue()) {
densityAvgSum += ((double) diff(newData.getHighValue(), newData.getLowValue())) / newData.getNumDVs();
}
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
ndvEstimator.mergeEstimators(newData.getNdvEstimator());
}
if (aggregateData == null) {
aggregateData = newData.deepCopy();
} else {
- DateColumnStatsMerger merger = new DateColumnStatsMerger();
merger.setLowValue(aggregateData, newData);
merger.setHighValue(aggregateData, newData);
@@ -120,7 +119,7 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
}
}
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
// if all the ColumnStatisticsObjs contain bitvectors, we do not need to
// use uniform distribution assumption because we can merge bitvectors
// to get a good estimation.
@@ -143,11 +142,12 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
}
aggregateData.setNumDVs(estimation);
}
+
columnStatisticsData.setDateStats(aggregateData);
} else {
+ // TODO: bail out if missing stats are over a certain threshold
// we need extrapolation
- LOG.debug("start extrapolation for " + colName);
-
+ LOG.debug("start extrapolation for {}", colName);
Map<String, Integer> indexMap = new HashMap<>();
for (int index = 0; index < partNames.size(); index++) {
indexMap.put(partNames.get(index), index);
@@ -155,16 +155,16 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
Map<String, Double> adjustedIndexMap = new HashMap<>();
Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<>();
// while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
+ // higherbound when useDensityFunctionForNDVEstimation is true.
double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
+ if (!areAllNDVEstimatorsMergeable) {
// if not every partition uses bitvector for ndv, we just fall back to
// the traditional extrapolation methods.
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
String partName = csp.getPartName();
DateColumnStatsData newData = cso.getStatsData().getDateStats();
- if (useDensityFunctionForNDVEstimation) {
+ if (useDensityFunctionForNDVEstimation && newData.isSetLowValue() && newData.isSetHighValue()) {
densityAvgSum += ((double) diff(newData.getHighValue(), newData.getLowValue())) / newData.getNumDVs();
}
adjustedIndexMap.put(partName, (double) indexMap.get(partName));
@@ -236,13 +236,28 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
adjustedIndexMap, adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
}
LOG.debug(
- "Ndv estimatation for {} is {} # of partitions requested: {} # of partitions found: {}",
+ "Ndv estimation for {} is {}. # of partitions requested: {}. # of partitions found: {}",
colName, columnStatisticsData.getDateStats().getNumDVs(), partNames.size(),
colStatsWithSourceInfo.size());
+
+ KllHistogramEstimator mergedKllHistogramEstimator = mergeHistograms(colStatsWithSourceInfo);
+ if (mergedKllHistogramEstimator != null) {
+ columnStatisticsData.getDateStats().setHistogram(mergedKllHistogramEstimator.serialize());
+ }
+
statsObj.setStatsData(columnStatisticsData);
return statsObj;
}
+ @Override protected ColumnStatisticsData initColumnStatisticsData() {
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ // init stats internal data if missing, re-use if existing
+ if (!columnStatisticsData.isSetDateStats()) {
+ columnStatisticsData.setDateStats(new DateColumnStatsData());
+ }
+ return columnStatisticsData;
+ }
+
private long diff(Date d1, Date d2) {
return d1.getDaysSinceEpoch() - d2.getDaysSinceEpoch();
}
@@ -259,8 +274,6 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
int numPartsWithStats, Map<String, Double> adjustedIndexMap,
Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- DateColumnStatsDataInspector extrapolateDateData = new DateColumnStatsDataInspector();
Map<String, DateColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDateStats());
@@ -268,16 +281,10 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
List<Map.Entry<String, DateColumnStatsData>> list = new LinkedList<>(
extractedAdjustedStatsMap.entrySet());
// get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, DateColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DateColumnStatsData> o1,
- Map.Entry<String, DateColumnStatsData> o2) {
- return o1.getValue().getLowValue().compareTo(o2.getValue().getLowValue());
- }
- });
+ list.sort(Comparator.comparing(o -> o.getValue().getLowValue()));
double minInd = adjustedIndexMap.get(list.get(0).getKey());
double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long lowValue = 0;
+ long lowValue;
long min = list.get(0).getValue().getLowValue().getDaysSinceEpoch();
long max = list.get(list.size() - 1).getValue().getLowValue().getDaysSinceEpoch();
if (minInd == maxInd) {
@@ -287,27 +294,21 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
} else {
// right border is the min
- lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ lowValue = (long) (max - (max - min) * (numParts - maxInd) / (minInd - maxInd));
}
// get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, DateColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DateColumnStatsData> o1,
- Map.Entry<String, DateColumnStatsData> o2) {
- return o1.getValue().getHighValue().compareTo(o2.getValue().getHighValue());
- }
- });
+ list.sort(Comparator.comparing(o -> o.getValue().getHighValue()));
minInd = adjustedIndexMap.get(list.get(0).getKey());
maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long highValue = 0;
+ long highValue;
min = list.get(0).getValue().getHighValue().getDaysSinceEpoch();
max = list.get(list.size() - 1).getValue().getHighValue().getDaysSinceEpoch();
if (minInd == maxInd) {
highValue = min;
} else if (minInd < maxInd) {
// right border is the max
- highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ highValue = (long) (min + (max - min) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
@@ -322,14 +323,8 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
numNulls = numNulls * numParts / numPartsWithStats;
// get the ndv
- long ndv = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, DateColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DateColumnStatsData> o1,
- Map.Entry<String, DateColumnStatsData> o2) {
- return Long.compare(o1.getValue().getNumDVs(), o2.getValue().getNumDVs());
- }
- });
+ long ndv;
+ list.sort(Comparator.comparingLong(o -> o.getValue().getNumDVs()));
long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
long higherBound = 0;
for (Map.Entry<String, DateColumnStatsData> entry : list) {
@@ -351,12 +346,13 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
ndv = min;
} else if (minInd < maxInd) {
// right border is the max
- ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ ndv = (long) (min + (max - min) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
}
}
+ DateColumnStatsDataInspector extrapolateDateData = new DateColumnStatsDataInspector();
extrapolateDateData.setLowValue(new Date(lowValue));
extrapolateDateData.setHighValue(new Date(highValue));
extrapolateDateData.setNumNulls(numNulls);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java
index 3e2093829b7..7d7e6251c64 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java
@@ -19,13 +19,13 @@
package org.apache.hadoop.hive.metastore.columnstats.aggr;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
@@ -53,12 +53,13 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
checkStatisticsList(colStatsWithSourceInfo);
ColumnStatisticsObj statsObj = null;
- String colType = null;
+ String colType;
String colName = null;
// check if all the ColumnStatisticsObjs contain stats and all the ndv are
// bitvectors
boolean doAllPartitionContainStats = partNames.size() == colStatsWithSourceInfo.size();
NumDistinctValueEstimator ndvEstimator = null;
+ boolean areAllNDVEstimatorsMergeable = true;
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
if (statsObj == null) {
@@ -69,37 +70,36 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName,
doAllPartitionContainStats);
}
- DecimalColumnStatsDataInspector decimalColumnStatsData = decimalInspectorFromStats(cso);
+ DecimalColumnStatsDataInspector columnStatsData = decimalInspectorFromStats(cso);
- if (decimalColumnStatsData.getNdvEstimator() == null) {
- ndvEstimator = null;
+ // check if we can merge NDV estimators
+ if (columnStatsData.getNdvEstimator() == null) {
+ areAllNDVEstimatorsMergeable = false;
break;
} else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = decimalColumnStatsData.getNdvEstimator();
+ NumDistinctValueEstimator estimator = columnStatsData.getNdvEstimator();
if (ndvEstimator == null) {
ndvEstimator = estimator;
} else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
+ if (!ndvEstimator.canMerge(estimator)) {
+ areAllNDVEstimatorsMergeable = false;
break;
}
}
}
}
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
+ ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
}
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ LOG.debug("all of the bit vectors can merge for {} is {}", colName, areAllNDVEstimatorsMergeable);
+
+ ColumnStatisticsData columnStatisticsData = initColumnStatisticsData();
if (doAllPartitionContainStats || colStatsWithSourceInfo.size() < 2) {
DecimalColumnStatsDataInspector aggregateData = null;
long lowerBound = 0;
long higherBound = 0;
double densityAvgSum = 0.0;
+ DecimalColumnStatsMerger merger = new DecimalColumnStatsMerger();
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
DecimalColumnStatsDataInspector newData = decimalInspectorFromStats(cso);
@@ -109,13 +109,12 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
densityAvgSum += (MetaStoreServerUtils.decimalToDouble(newData.getHighValue()) - MetaStoreServerUtils
.decimalToDouble(newData.getLowValue())) / newData.getNumDVs();
}
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
ndvEstimator.mergeEstimators(newData.getNdvEstimator());
}
if (aggregateData == null) {
aggregateData = newData.deepCopy();
} else {
- DecimalColumnStatsMerger merger = new DecimalColumnStatsMerger();
merger.setLowValue(aggregateData, newData);
merger.setHighValue(aggregateData, newData);
@@ -123,7 +122,7 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
}
}
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
// if all the ColumnStatisticsObjs contain bitvectors, we do not need to
// use uniform distribution assumption because we can merge bitvectors
// to get a good estimation.
@@ -147,10 +146,12 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
}
aggregateData.setNumDVs(estimation);
}
+
columnStatisticsData.setDecimalStats(aggregateData);
} else {
+ // TODO: bail out if missing stats are over a certain threshold
// we need extrapolation
- LOG.debug("start extrapolation for " + colName);
+ LOG.debug("start extrapolation for {}", colName);
Map<String, Integer> indexMap = new HashMap<>();
for (int index = 0; index < partNames.size(); index++) {
indexMap.put(partNames.get(index), index);
@@ -158,16 +159,16 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
Map<String, Double> adjustedIndexMap = new HashMap<>();
Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<>();
// while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
+ // higherbound when useDensityFunctionForNDVEstimation is true.
double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
+ if (!areAllNDVEstimatorsMergeable) {
// if not every partition uses bitvector for ndv, we just fall back to
// the traditional extrapolation methods.
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
String partName = csp.getPartName();
DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
- if (useDensityFunctionForNDVEstimation) {
+ if (useDensityFunctionForNDVEstimation && newData.isSetLowValue() && newData.isSetHighValue()) {
densityAvgSum += (MetaStoreServerUtils.decimalToDouble(newData.getHighValue()) - MetaStoreServerUtils
.decimalToDouble(newData.getLowValue())) / newData.getNumDVs();
}
@@ -250,19 +251,33 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
adjustedIndexMap, adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
}
LOG.debug(
- "Ndv estimatation for {} is {} # of partitions requested: {} # of partitions found: {}",
+ "Ndv estimation for {} is {}. # of partitions requested: {}. # of partitions found: {}",
colName, columnStatisticsData.getDecimalStats().getNumDVs(), partNames.size(),
colStatsWithSourceInfo.size());
+
+ KllHistogramEstimator mergedKllHistogramEstimator = mergeHistograms(colStatsWithSourceInfo);
+ if (mergedKllHistogramEstimator != null) {
+ columnStatisticsData.getDecimalStats().setHistogram(mergedKllHistogramEstimator.serialize());
+ }
+
statsObj.setStatsData(columnStatisticsData);
return statsObj;
}
+ @Override protected ColumnStatisticsData initColumnStatisticsData() {
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ // init stats internal data if missing, re-use if existing
+ if (!columnStatisticsData.isSetDecimalStats()) {
+ columnStatisticsData.setDecimalStats(new DecimalColumnStatsData());
+ }
+ return columnStatisticsData;
+ }
+
@Override
public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
int numPartsWithStats, Map<String, Double> adjustedIndexMap,
Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- DecimalColumnStatsDataInspector extrapolateDecimalData = new DecimalColumnStatsDataInspector();
+
Map<String, DecimalColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDecimalStats());
@@ -270,16 +285,10 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
List<Map.Entry<String, DecimalColumnStatsData>> list = new LinkedList<>(
extractedAdjustedStatsMap.entrySet());
// get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
- Map.Entry<String, DecimalColumnStatsData> o2) {
- return o1.getValue().getLowValue().compareTo(o2.getValue().getLowValue());
- }
- });
+ list.sort(Comparator.comparing(o -> o.getValue().getLowValue()));
double minInd = adjustedIndexMap.get(list.get(0).getKey());
double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double lowValue = 0;
+ double lowValue;
double min = MetaStoreServerUtils.decimalToDouble(list.get(0).getValue().getLowValue());
double max = MetaStoreServerUtils.decimalToDouble(list.get(list.size() - 1).getValue().getLowValue());
if (minInd == maxInd) {
@@ -289,27 +298,21 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
} else {
// right border is the min
- lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ lowValue = (max - (max - min) * (numParts - maxInd) / (minInd - maxInd));
}
// get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
- Map.Entry<String, DecimalColumnStatsData> o2) {
- return o1.getValue().getHighValue().compareTo(o2.getValue().getHighValue());
- }
- });
+ list.sort(Comparator.comparing(o -> o.getValue().getHighValue()));
minInd = adjustedIndexMap.get(list.get(0).getKey());
maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double highValue = 0;
+ double highValue;
min = MetaStoreServerUtils.decimalToDouble(list.get(0).getValue().getHighValue());
max = MetaStoreServerUtils.decimalToDouble(list.get(list.size() - 1).getValue().getHighValue());
if (minInd == maxInd) {
highValue = min;
} else if (minInd < maxInd) {
// right border is the max
- highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ highValue = (min + (max - min) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
highValue = (min + (max - min) * minInd / (minInd - maxInd));
@@ -324,16 +327,10 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
numNulls = numNulls * numParts / numPartsWithStats;
// get the ndv
- long ndv = 0;
- long ndvMin = 0;
- long ndvMax = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
- Map.Entry<String, DecimalColumnStatsData> o2) {
- return Long.compare(o1.getValue().getNumDVs(), o2.getValue().getNumDVs());
- }
- });
+ long ndv;
+ long ndvMin;
+ long ndvMax;
+ list.sort(Comparator.comparingLong(o -> o.getValue().getNumDVs()));
long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
long higherBound = 0;
for (Map.Entry<String, DecimalColumnStatsData> entry : list) {
@@ -355,18 +352,16 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen
ndv = ndvMin;
} else if (minInd < maxInd) {
// right border is the max
- ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
}
}
- extrapolateDecimalData.setLowValue(DecimalUtils.createThriftDecimal(String
- .valueOf(lowValue)));
- extrapolateDecimalData.setHighValue(DecimalUtils.createThriftDecimal(String
- .valueOf(highValue)));
+ DecimalColumnStatsData extrapolateDecimalData = extrapolateData.getDecimalStats();
+ extrapolateDecimalData.setLowValue(DecimalUtils.createThriftDecimal(String.valueOf(lowValue)));
+ extrapolateDecimalData.setHighValue(DecimalUtils.createThriftDecimal(String.valueOf(highValue)));
extrapolateDecimalData.setNumNulls(numNulls);
extrapolateDecimalData.setNumDVs(ndv);
- extrapolateData.setDecimalStats(extrapolateDecimalData);
}
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java
index 2caa2f32a3c..2ce2c7281aa 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java
@@ -19,13 +19,13 @@
package org.apache.hadoop.hive.metastore.columnstats.aggr;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
@@ -43,7 +43,7 @@ import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.dou
public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implements
IExtrapolatePartStatus {
- private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
+ private static final Logger LOG = LoggerFactory.getLogger(DoubleColumnStatsAggregator.class);
@Override
public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo,
@@ -51,12 +51,13 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
checkStatisticsList(colStatsWithSourceInfo);
ColumnStatisticsObj statsObj = null;
- String colType = null;
+ String colType;
String colName = null;
// check if all the ColumnStatisticsObjs contain stats and all the ndv are
// bitvectors
boolean doAllPartitionContainStats = partNames.size() == colStatsWithSourceInfo.size();
NumDistinctValueEstimator ndvEstimator = null;
+ boolean areAllNDVEstimatorsMergeable = true;
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
if (statsObj == null) {
@@ -67,50 +68,48 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName,
doAllPartitionContainStats);
}
- DoubleColumnStatsDataInspector doubleColumnStatsData =
- doubleInspectorFromStats(cso);
- if (doubleColumnStatsData.getNdvEstimator() == null) {
- ndvEstimator = null;
+ DoubleColumnStatsDataInspector columnStatsData = doubleInspectorFromStats(cso);
+
+ // check if we can merge NDV estimators
+ if (columnStatsData.getNdvEstimator() == null) {
+ areAllNDVEstimatorsMergeable = false;
break;
} else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = doubleColumnStatsData.getNdvEstimator();
+ NumDistinctValueEstimator estimator = columnStatsData.getNdvEstimator();
if (ndvEstimator == null) {
ndvEstimator = estimator;
} else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
+ if (!ndvEstimator.canMerge(estimator)) {
+ areAllNDVEstimatorsMergeable = false;
break;
}
}
}
}
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
+ ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
}
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ LOG.debug("all of the bit vectors can merge for {} is {}", colName, areAllNDVEstimatorsMergeable);
+
+ ColumnStatisticsData columnStatisticsData = initColumnStatisticsData();
if (doAllPartitionContainStats || colStatsWithSourceInfo.size() < 2) {
DoubleColumnStatsDataInspector aggregateData = null;
long lowerBound = 0;
long higherBound = 0;
double densityAvgSum = 0.0;
+ DoubleColumnStatsMerger merger = new DoubleColumnStatsMerger();
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
DoubleColumnStatsDataInspector newData = doubleInspectorFromStats(cso);
lowerBound = Math.max(lowerBound, newData.getNumDVs());
higherBound += newData.getNumDVs();
densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
ndvEstimator.mergeEstimators(newData.getNdvEstimator());
}
if (aggregateData == null) {
aggregateData = newData.deepCopy();
} else {
- DoubleColumnStatsMerger merger = new DoubleColumnStatsMerger();
merger.setLowValue(aggregateData, newData);
merger.setHighValue(aggregateData, newData);
@@ -118,7 +117,7 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
}
}
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
// if all the ColumnStatisticsObjs contain bitvectors, we do not need to
// use uniform distribution assumption because we can merge bitvectors
// to get a good estimation.
@@ -140,10 +139,12 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
}
aggregateData.setNumDVs(estimation);
}
+
columnStatisticsData.setDoubleStats(aggregateData);
} else {
+ // TODO: bail out if missing stats are over a certain threshold
// we need extrapolation
- LOG.debug("start extrapolation for " + colName);
+ LOG.debug("start extrapolation for {}", colName);
Map<String, Integer> indexMap = new HashMap<>();
for (int index = 0; index < partNames.size(); index++) {
indexMap.put(partNames.get(index), index);
@@ -151,9 +152,9 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
Map<String, Double> adjustedIndexMap = new HashMap<>();
Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<>();
// while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
+ // higherbound when useDensityFunctionForNDVEstimation is true.
double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
+ if (!areAllNDVEstimatorsMergeable) {
// if not every partition uses bitvector for ndv, we just fall back to
// the traditional extrapolation methods.
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
@@ -232,19 +233,33 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
adjustedIndexMap, adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
}
LOG.debug(
- "Ndv estimatation for {} is {}. # of partitions requested: {}. # of partitions found: {}",
+ "Ndv estimation for {} is {}. # of partitions requested: {}. # of partitions found: {}",
colName, columnStatisticsData.getDoubleStats().getNumDVs(), partNames.size(),
colStatsWithSourceInfo.size());
+
+ KllHistogramEstimator mergedKllHistogramEstimator = mergeHistograms(colStatsWithSourceInfo);
+ if (mergedKllHistogramEstimator != null) {
+ columnStatisticsData.getDoubleStats().setHistogram(mergedKllHistogramEstimator.serialize());
+ }
+
statsObj.setStatsData(columnStatisticsData);
return statsObj;
}
+ @Override protected ColumnStatisticsData initColumnStatisticsData() {
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ // init stats internal data if missing, re-use if existing
+ if (!columnStatisticsData.isSetDoubleStats()) {
+ columnStatisticsData.setDoubleStats(new DoubleColumnStatsData());
+ }
+ return columnStatisticsData;
+ }
+
@Override
public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
int numPartsWithStats, Map<String, Double> adjustedIndexMap,
Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- DoubleColumnStatsDataInspector extrapolateDoubleData = new DoubleColumnStatsDataInspector();
+
Map<String, DoubleColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDoubleStats());
@@ -252,16 +267,10 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
List<Map.Entry<String, DoubleColumnStatsData>> list = new LinkedList<>(
extractedAdjustedStatsMap.entrySet());
// get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
- Map.Entry<String, DoubleColumnStatsData> o2) {
- return Double.compare(o1.getValue().getLowValue(), o2.getValue().getLowValue());
- }
- });
+ list.sort(Comparator.comparingDouble(o -> o.getValue().getLowValue()));
double minInd = adjustedIndexMap.get(list.get(0).getKey());
double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double lowValue = 0;
+ double lowValue;
double min = list.get(0).getValue().getLowValue();
double max = list.get(list.size() - 1).getValue().getLowValue();
if (minInd == maxInd) {
@@ -271,27 +280,21 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
} else {
// right border is the min
- lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ lowValue = (max - (max - min) * (numParts - maxInd) / (minInd - maxInd));
}
// get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
- Map.Entry<String, DoubleColumnStatsData> o2) {
- return Double.compare(o1.getValue().getHighValue(), o2.getValue().getHighValue());
- }
- });
+ list.sort(Comparator.comparingDouble(o -> o.getValue().getHighValue()));
minInd = adjustedIndexMap.get(list.get(0).getKey());
maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double highValue = 0;
+ double highValue;
min = list.get(0).getValue().getHighValue();
max = list.get(list.size() - 1).getValue().getHighValue();
if (minInd == maxInd) {
highValue = min;
} else if (minInd < maxInd) {
// right border is the max
- highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ highValue = (min + (max - min) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
highValue = (min + (max - min) * minInd / (minInd - maxInd));
@@ -306,16 +309,10 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
numNulls = numNulls * numParts / numPartsWithStats;
// get the ndv
- long ndv = 0;
- long ndvMin = 0;
- long ndvMax = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
- Map.Entry<String, DoubleColumnStatsData> o2) {
- return Long.compare(o1.getValue().getNumDVs(), o2.getValue().getNumDVs());
- }
- });
+ long ndv;
+ long ndvMin;
+ long ndvMax;
+ list.sort(Comparator.comparingLong(o -> o.getValue().getNumDVs()));
long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
long higherBound = 0;
for (Map.Entry<String, DoubleColumnStatsData> entry : list) {
@@ -337,17 +334,16 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement
ndv = ndvMin;
} else if (minInd < maxInd) {
// right border is the max
- ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
}
}
+ DoubleColumnStatsData extrapolateDoubleData = extrapolateData.getDoubleStats();
extrapolateDoubleData.setLowValue(lowValue);
extrapolateDoubleData.setHighValue(highValue);
extrapolateDoubleData.setNumNulls(numNulls);
extrapolateDoubleData.setNumDVs(ndv);
- extrapolateData.setDoubleStats(extrapolateDoubleData);
}
-
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java
index dd35e0b35c8..d2999054a53 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java
@@ -19,13 +19,13 @@
package org.apache.hadoop.hive.metastore.columnstats.aggr;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
@@ -51,12 +51,13 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
checkStatisticsList(colStatsWithSourceInfo);
ColumnStatisticsObj statsObj = null;
- String colType = null;
+ String colType;
String colName = null;
// check if all the ColumnStatisticsObjs contain stats and all the ndv are
// bitvectors
boolean doAllPartitionContainStats = partNames.size() == colStatsWithSourceInfo.size();
NumDistinctValueEstimator ndvEstimator = null;
+ boolean areAllNDVEstimatorsMergeable = true;
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
if (statsObj == null) {
@@ -64,52 +65,50 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
colType = cso.getColType();
statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType,
cso.getStatsData().getSetField());
- LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName,
- doAllPartitionContainStats);
+ LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, doAllPartitionContainStats);
}
- LongColumnStatsDataInspector longColumnStatsData = longInspectorFromStats(cso);
- if (longColumnStatsData.getNdvEstimator() == null) {
- ndvEstimator = null;
+ LongColumnStatsDataInspector columnStatsData = longInspectorFromStats(cso);
+
+ // check if we can merge NDV estimators
+ if (columnStatsData.getNdvEstimator() == null) {
+ areAllNDVEstimatorsMergeable = false;
break;
} else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = longColumnStatsData.getNdvEstimator();
+ NumDistinctValueEstimator estimator = columnStatsData.getNdvEstimator();
if (ndvEstimator == null) {
ndvEstimator = estimator;
} else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
+ if (!ndvEstimator.canMerge(estimator)) {
+ areAllNDVEstimatorsMergeable = false;
break;
}
}
}
}
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
+ ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
}
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ LOG.debug("all of the bit vectors can merge for {} is {}", colName, areAllNDVEstimatorsMergeable);
+
+ ColumnStatisticsData columnStatisticsData = initColumnStatisticsData();
if (doAllPartitionContainStats || colStatsWithSourceInfo.size() < 2) {
LongColumnStatsDataInspector aggregateData = null;
long lowerBound = 0;
long higherBound = 0;
double densityAvgSum = 0.0;
+ LongColumnStatsMerger merger = new LongColumnStatsMerger();
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
LongColumnStatsDataInspector newData = longInspectorFromStats(cso);
lowerBound = Math.max(lowerBound, newData.getNumDVs());
higherBound += newData.getNumDVs();
densityAvgSum += ((double) (newData.getHighValue() - newData.getLowValue())) / newData.getNumDVs();
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
ndvEstimator.mergeEstimators(newData.getNdvEstimator());
}
if (aggregateData == null) {
aggregateData = newData.deepCopy();
} else {
- LongColumnStatsMerger merger = new LongColumnStatsMerger();
merger.setLowValue(aggregateData, newData);
merger.setHighValue(aggregateData, newData);
@@ -117,7 +116,7 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
}
}
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
// if all the ColumnStatisticsObjs contain bitvectors, we do not need to
// use uniform distribution assumption because we can merge bitvectors
// to get a good estimation.
@@ -139,11 +138,12 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
}
aggregateData.setNumDVs(estimation);
}
+
columnStatisticsData.setLongStats(aggregateData);
} else {
+ // TODO: bail out if missing stats are over a certain threshold
// we need extrapolation
- LOG.debug("start extrapolation for " + colName);
-
+ LOG.debug("start extrapolation for {}", colName);
Map<String, Integer> indexMap = new HashMap<>();
for (int index = 0; index < partNames.size(); index++) {
indexMap.put(partNames.get(index), index);
@@ -151,9 +151,9 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
Map<String, Double> adjustedIndexMap = new HashMap<>();
Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<>();
// while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
+ // higherbound when useDensityFunctionForNDVEstimation is true.
double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
+ if (!areAllNDVEstimatorsMergeable) {
// if not every partition uses bitvector for ndv, we just fall back to
// the traditional extrapolation methods.
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
@@ -209,8 +209,7 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
aggregateData = newData.deepCopy();
} else {
aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
- newData.getHighValue()));
+ aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
}
ndvEstimator.mergeEstimators(newData.getNdvEstimator());
@@ -231,19 +230,33 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
adjustedIndexMap, adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
}
LOG.debug(
- "Ndv estimatation for {} is {} # of partitions requested: {} # of partitions found: {}",
+ "Ndv estimation for {} is {}. # of partitions requested: {}. # of partitions found: {}",
colName, columnStatisticsData.getLongStats().getNumDVs(), partNames.size(),
colStatsWithSourceInfo.size());
+
+ KllHistogramEstimator mergedKllHistogramEstimator = mergeHistograms(colStatsWithSourceInfo);
+ if (mergedKllHistogramEstimator != null) {
+ columnStatisticsData.getLongStats().setHistogram(mergedKllHistogramEstimator.serialize());
+ }
+
statsObj.setStatsData(columnStatisticsData);
return statsObj;
}
+ @Override protected ColumnStatisticsData initColumnStatisticsData() {
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ // init stats internal data if missing, re-use if existing
+ if (!columnStatisticsData.isSetLongStats()) {
+ columnStatisticsData.setLongStats(new LongColumnStatsData());
+ }
+ return columnStatisticsData;
+ }
+
@Override
public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
int numPartsWithStats, Map<String, Double> adjustedIndexMap,
Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- LongColumnStatsDataInspector extrapolateLongData = new LongColumnStatsDataInspector();
+
Map<String, LongColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getLongStats());
@@ -251,16 +264,10 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
List<Map.Entry<String, LongColumnStatsData>> list = new LinkedList<>(
extractedAdjustedStatsMap.entrySet());
// get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, LongColumnStatsData> o1,
- Map.Entry<String, LongColumnStatsData> o2) {
- return Long.compare(o1.getValue().getLowValue(), o2.getValue().getLowValue());
- }
- });
+ list.sort(Comparator.comparingLong(o -> o.getValue().getLowValue()));
double minInd = adjustedIndexMap.get(list.get(0).getKey());
double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long lowValue = 0;
+ long lowValue;
long min = list.get(0).getValue().getLowValue();
long max = list.get(list.size() - 1).getValue().getLowValue();
if (minInd == maxInd) {
@@ -270,27 +277,21 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
} else {
// right border is the min
- lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ lowValue = (long) (max - (max - min) * (numParts - maxInd) / (minInd - maxInd));
}
// get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, LongColumnStatsData> o1,
- Map.Entry<String, LongColumnStatsData> o2) {
- return Long.compare(o1.getValue().getHighValue(), o2.getValue().getHighValue());
- }
- });
+ list.sort(Comparator.comparingLong(o -> o.getValue().getHighValue()));
minInd = adjustedIndexMap.get(list.get(0).getKey());
maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long highValue = 0;
+ long highValue;
min = list.get(0).getValue().getHighValue();
max = list.get(list.size() - 1).getValue().getHighValue();
if (minInd == maxInd) {
highValue = min;
} else if (minInd < maxInd) {
// right border is the max
- highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ highValue = (long) (min + (max - min) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
@@ -305,14 +306,8 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
numNulls = numNulls * numParts / numPartsWithStats;
// get the ndv
- long ndv = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, LongColumnStatsData> o1,
- Map.Entry<String, LongColumnStatsData> o2) {
- return Long.compare(o1.getValue().getNumDVs(), o2.getValue().getNumDVs());
- }
- });
+ long ndv;
+ list.sort(Comparator.comparingLong(o -> o.getValue().getNumDVs()));
long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
long higherBound = 0;
for (Map.Entry<String, LongColumnStatsData> entry : list) {
@@ -334,17 +329,16 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
ndv = min;
} else if (minInd < maxInd) {
// right border is the max
- ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ ndv = (long) (min + (max - min) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
}
}
+ LongColumnStatsData extrapolateLongData = extrapolateData.getLongStats();
extrapolateLongData.setLowValue(lowValue);
extrapolateLongData.setHighValue(highValue);
extrapolateLongData.setNumNulls(numNulls);
extrapolateLongData.setNumDVs(ndv);
- extrapolateData.setLongStats(extrapolateLongData);
}
-
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java
index bb38b8cfaa7..98bb00416cd 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java
@@ -90,7 +90,7 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement
.getEmptyNumDistinctValueEstimator(ndvEstimator);
}
LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ ColumnStatisticsData columnStatisticsData = initColumnStatisticsData();
if (doAllPartitionContainStats || colStatsWithSourceInfo.size() < 2) {
StringColumnStatsDataInspector aggregateData = null;
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
@@ -120,6 +120,7 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement
}
columnStatisticsData.setStringStats(aggregateData);
} else {
+ // TODO: bail out if missing stats are over a certain threshold
// we need extrapolation
LOG.debug("start extrapolation for " + colName);
@@ -207,6 +208,15 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement
return statsObj;
}
+ @Override protected ColumnStatisticsData initColumnStatisticsData() {
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ // init stats internal data if missing, re-use if existing
+ if (!columnStatisticsData.isSetStringStats()) {
+ columnStatisticsData.setStringStats(new StringColumnStatsData());
+ }
+ return columnStatisticsData;
+ }
+
@Override
public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
int numPartsWithStats, Map<String, Double> adjustedIndexMap,
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/TimestampColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/TimestampColumnStatsAggregator.java
index 95e8db9fdf8..5962792cb5b 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/TimestampColumnStatsAggregator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/TimestampColumnStatsAggregator.java
@@ -19,13 +19,14 @@
package org.apache.hadoop.hive.metastore.columnstats.aggr;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimatorFactory;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
@@ -52,12 +53,13 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
checkStatisticsList(colStatsWithSourceInfo);
ColumnStatisticsObj statsObj = null;
- String colType = null;
+ String colType;
String colName = null;
// check if all the ColumnStatisticsObjs contain stats and all the ndv are
// bitvectors
boolean doAllPartitionContainStats = partNames.size() == colStatsWithSourceInfo.size();
NumDistinctValueEstimator ndvEstimator = null;
+ boolean areAllNDVEstimatorsMergeable = true;
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
if (statsObj == null) {
@@ -67,52 +69,50 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
cso.getStatsData().getSetField());
LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, doAllPartitionContainStats);
}
- TimestampColumnStatsDataInspector timestampColumnStats = timestampInspectorFromStats(cso);
+ TimestampColumnStatsDataInspector columnStatsData = timestampInspectorFromStats(cso);
- if (timestampColumnStats.getNdvEstimator() == null) {
- ndvEstimator = null;
+ // check if we can merge NDV estimators
+ if (columnStatsData.getNdvEstimator() == null) {
+ areAllNDVEstimatorsMergeable = false;
break;
} else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = timestampColumnStats.getNdvEstimator();
+ NumDistinctValueEstimator estimator = columnStatsData.getNdvEstimator();
if (ndvEstimator == null) {
ndvEstimator = estimator;
} else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
+ if (!ndvEstimator.canMerge(estimator)) {
+ areAllNDVEstimatorsMergeable = false;
break;
}
}
}
}
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
+ ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
}
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ LOG.debug("all of the bit vectors can merge for {} is {}", colName, areAllNDVEstimatorsMergeable);
+
+ ColumnStatisticsData columnStatisticsData = initColumnStatisticsData();
if (doAllPartitionContainStats || colStatsWithSourceInfo.size() < 2) {
TimestampColumnStatsDataInspector aggregateData = null;
long lowerBound = 0;
long higherBound = 0;
double densityAvgSum = 0.0;
+ TimestampColumnStatsMerger merger = new TimestampColumnStatsMerger();
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
TimestampColumnStatsDataInspector newData = timestampInspectorFromStats(cso);
lowerBound = Math.max(lowerBound, newData.getNumDVs());
higherBound += newData.getNumDVs();
if (newData.isSetLowValue() && newData.isSetHighValue()) {
- densityAvgSum += ((double) (diff(newData.getHighValue(), newData.getLowValue())) / newData.getNumDVs());
+ densityAvgSum += ((double) diff(newData.getHighValue(), newData.getLowValue())) / newData.getNumDVs();
}
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
ndvEstimator.mergeEstimators(newData.getNdvEstimator());
}
if (aggregateData == null) {
aggregateData = newData.deepCopy();
} else {
- TimestampColumnStatsMerger merger = new TimestampColumnStatsMerger();
merger.setLowValue(aggregateData, newData);
merger.setHighValue(aggregateData, newData);
@@ -120,7 +120,7 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
}
}
- if (ndvEstimator != null) {
+ if (areAllNDVEstimatorsMergeable && ndvEstimator != null) {
// if all the ColumnStatisticsObjs contain bitvectors, we do not need to
// use uniform distribution assumption because we can merge bitvectors
// to get a good estimation.
@@ -143,11 +143,12 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
}
aggregateData.setNumDVs(estimation);
}
+
columnStatisticsData.setTimestampStats(aggregateData);
} else {
+ // TODO: bail out if missing stats are over a certain threshold
// we need extrapolation
- LOG.debug("start extrapolation for " + colName);
-
+ LOG.debug("start extrapolation for {}", colName);
Map<String, Integer> indexMap = new HashMap<>();
for (int index = 0; index < partNames.size(); index++) {
indexMap.put(partNames.get(index), index);
@@ -155,17 +156,17 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
Map<String, Double> adjustedIndexMap = new HashMap<>();
Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<>();
// while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
+ // higherbound when useDensityFunctionForNDVEstimation is true.
double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
+ if (!areAllNDVEstimatorsMergeable) {
// if not every partition uses bitvector for ndv, we just fall back to
// the traditional extrapolation methods.
for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
ColumnStatisticsObj cso = csp.getColStatsObj();
String partName = csp.getPartName();
TimestampColumnStatsData newData = cso.getStatsData().getTimestampStats();
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += ((double) diff(newData.getHighValue(), newData.getLowValue()) / newData.getNumDVs());
+ if (useDensityFunctionForNDVEstimation && newData.isSetLowValue() && newData.isSetHighValue()) {
+ densityAvgSum += ((double) diff(newData.getHighValue(), newData.getLowValue())) / newData.getNumDVs();
}
adjustedIndexMap.put(partName, (double) indexMap.get(partName));
adjustedStatsMap.put(partName, cso.getStatsData());
@@ -236,13 +237,28 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
adjustedIndexMap, adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
}
LOG.debug(
- "Ndv estimatation for {} is {} # of partitions requested: {} # of partitions found: {}",
+ "Ndv estimation for {} is {}. # of partitions requested: {}. # of partitions found: {}",
colName, columnStatisticsData.getTimestampStats().getNumDVs(), partNames.size(),
colStatsWithSourceInfo.size());
+
+ KllHistogramEstimator mergedKllHistogramEstimator = mergeHistograms(colStatsWithSourceInfo);
+ if (mergedKllHistogramEstimator != null) {
+ columnStatisticsData.getTimestampStats().setHistogram(mergedKllHistogramEstimator.serialize());
+ }
+
statsObj.setStatsData(columnStatisticsData);
return statsObj;
}
+ @Override protected ColumnStatisticsData initColumnStatisticsData() {
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ // init stats internal data if missing, re-use if existing
+ if (!columnStatisticsData.isSetTimestampStats()) {
+ columnStatisticsData.setTimestampStats(new TimestampColumnStatsData());
+ }
+ return columnStatisticsData;
+ }
+
private long diff(Timestamp d1, Timestamp d2) {
return d1.getSecondsSinceEpoch() - d2.getSecondsSinceEpoch();
}
@@ -259,8 +275,6 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
int numPartsWithStats, Map<String, Double> adjustedIndexMap,
Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- TimestampColumnStatsDataInspector extrapolateTimestampData = new TimestampColumnStatsDataInspector();
Map<String, TimestampColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getTimestampStats());
@@ -268,16 +282,10 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
List<Map.Entry<String, TimestampColumnStatsData>> list = new LinkedList<>(
extractedAdjustedStatsMap.entrySet());
// get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, TimestampColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, TimestampColumnStatsData> o1,
- Map.Entry<String, TimestampColumnStatsData> o2) {
- return o1.getValue().getLowValue().compareTo(o2.getValue().getLowValue());
- }
- });
+ list.sort(Comparator.comparing(o -> o.getValue().getLowValue()));
double minInd = adjustedIndexMap.get(list.get(0).getKey());
double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long lowValue = 0;
+ long lowValue;
long min = list.get(0).getValue().getLowValue().getSecondsSinceEpoch();
long max = list.get(list.size() - 1).getValue().getLowValue().getSecondsSinceEpoch();
if (minInd == maxInd) {
@@ -287,27 +295,21 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
} else {
// right border is the min
- lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ lowValue = (long) (max - (max - min) * (numParts - maxInd) / (minInd - maxInd));
}
// get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, TimestampColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, TimestampColumnStatsData> o1,
- Map.Entry<String, TimestampColumnStatsData> o2) {
- return o1.getValue().getHighValue().compareTo(o2.getValue().getHighValue());
- }
- });
+ list.sort(Comparator.comparing(o -> o.getValue().getHighValue()));
minInd = adjustedIndexMap.get(list.get(0).getKey());
maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long highValue = 0;
+ long highValue;
min = list.get(0).getValue().getHighValue().getSecondsSinceEpoch();
max = list.get(list.size() - 1).getValue().getHighValue().getSecondsSinceEpoch();
if (minInd == maxInd) {
highValue = min;
} else if (minInd < maxInd) {
// right border is the max
- highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ highValue = (long) (min + (max - min) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
@@ -322,14 +324,8 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
numNulls = numNulls * numParts / numPartsWithStats;
// get the ndv
- long ndv = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, TimestampColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, TimestampColumnStatsData> o1,
- Map.Entry<String, TimestampColumnStatsData> o2) {
- return Long.compare(o1.getValue().getNumDVs(), o2.getValue().getNumDVs());
- }
- });
+ long ndv;
+ list.sort(Comparator.comparingLong(o -> o.getValue().getNumDVs()));
long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
long higherBound = 0;
for (Map.Entry<String, TimestampColumnStatsData> entry : list) {
@@ -351,12 +347,13 @@ public class TimestampColumnStatsAggregator extends ColumnStatsAggregator implem
ndv = min;
} else if (minInd < maxInd) {
// right border is the max
- ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ ndv = (long) (min + (max - min) * (numParts - minInd) / (maxInd - minInd));
} else {
// left border is the max
ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
}
}
+ TimestampColumnStatsDataInspector extrapolateTimestampData = new TimestampColumnStatsDataInspector();
extrapolateTimestampData.setLowValue(new Timestamp(lowValue));
extrapolateTimestampData.setHighValue(new Timestamp(highValue));
extrapolateTimestampData.setNumNulls(numNulls);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DateColumnStatsDataInspector.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DateColumnStatsDataInspector.java
index 64fd7d66b80..8113297a912 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DateColumnStatsDataInspector.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DateColumnStatsDataInspector.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.metastore.columnstats.cache;
import java.nio.ByteBuffer;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimatorFactory;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
@@ -27,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
public class DateColumnStatsDataInspector extends DateColumnStatsData {
private NumDistinctValueEstimator ndvEstimator;
+ private KllHistogramEstimator histogramEstimator;
public DateColumnStatsDataInspector() {
super();
@@ -41,6 +44,9 @@ public class DateColumnStatsDataInspector extends DateColumnStatsData {
if (other.ndvEstimator != null) {
super.setBitVectors(ndvEstimator.serialize());
}
+ if (other.histogramEstimator != null) {
+ super.setHistogram(histogramEstimator.serialize());
+ }
}
public DateColumnStatsDataInspector(DateColumnStatsData other) {
@@ -60,6 +66,14 @@ public class DateColumnStatsDataInspector extends DateColumnStatsData {
return super.getBitVectors();
}
+ @Override
+ public byte[] getHistogram() {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ return super.getHistogram();
+ }
+
@Override
public ByteBuffer bufferForBitVectors() {
if (ndvEstimator != null) {
@@ -68,6 +82,14 @@ public class DateColumnStatsDataInspector extends DateColumnStatsData {
return super.bufferForBitVectors();
}
+ @Override
+ public ByteBuffer bufferForHistogram() {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ return super.bufferForHistogram();
+ }
+
@Override
public void setBitVectors(byte[] bitVectors) {
super.setBitVectors(bitVectors);
@@ -80,12 +102,30 @@ public class DateColumnStatsDataInspector extends DateColumnStatsData {
this.ndvEstimator = null;
}
+ @Override
+ public void setHistogram(byte[] stats) {
+ super.setHistogram(stats);
+ this.histogramEstimator = null;
+ }
+
+ @Override
+ public void setHistogram(ByteBuffer stats) {
+ super.setHistogram(stats);
+ this.histogramEstimator = null;
+ }
+
@Override
public void unsetBitVectors() {
super.unsetBitVectors();
this.ndvEstimator = null;
}
+ @Override
+ public void unsetHistogram() {
+ super.unsetHistogram();
+ this.histogramEstimator = null;
+ }
+
@Override
public boolean isSetBitVectors() {
if (ndvEstimator != null) {
@@ -94,6 +134,14 @@ public class DateColumnStatsDataInspector extends DateColumnStatsData {
return super.isSetBitVectors();
}
+ @Override
+ public boolean isSetHistogram() {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ return super.isSetHistogram();
+ }
+
@Override
public void setBitVectorsIsSet(boolean value) {
if (ndvEstimator != null) {
@@ -102,6 +150,14 @@ public class DateColumnStatsDataInspector extends DateColumnStatsData {
super.setBitVectorsIsSet(value);
}
+ @Override
+ public void setHistogramIsSet(boolean value) {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ super.setHistogramIsSet(value);
+ }
+
public NumDistinctValueEstimator getNdvEstimator() {
if (ndvEstimator == null && isSetBitVectors() && getBitVectors().length != 0) {
updateNdvEstimator();
@@ -109,20 +165,42 @@ public class DateColumnStatsDataInspector extends DateColumnStatsData {
return ndvEstimator;
}
+ public KllHistogramEstimator getHistogramEstimator() {
+ if (histogramEstimator == null && isSetHistogram() && getHistogram().length != 0) {
+ updateHistogramEstimator();
+ }
+ return histogramEstimator;
+ }
+
public void setNdvEstimator(NumDistinctValueEstimator ndvEstimator) {
super.unsetBitVectors();
this.ndvEstimator = ndvEstimator;
}
+ public void setHistogramEstimator(KllHistogramEstimator histogramEstimator) {
+ super.unsetHistogram();
+ this.histogramEstimator = histogramEstimator;
+ }
+
private void updateBitVectors() {
super.setBitVectors(ndvEstimator.serialize());
this.ndvEstimator = null;
}
+ private void updateHistogram() {
+ super.setHistogram(histogramEstimator.serialize());
+ this.histogramEstimator = null;
+ }
+
private void updateNdvEstimator() {
this.ndvEstimator = NumDistinctValueEstimatorFactory
.getNumDistinctValueEstimator(super.getBitVectors());
super.unsetBitVectors();
}
+ private void updateHistogramEstimator() {
+ this.histogramEstimator = KllHistogramEstimatorFactory
+ .getKllHistogramEstimator(super.getHistogram());
+ super.unsetHistogram();
+ }
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DecimalColumnStatsDataInspector.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DecimalColumnStatsDataInspector.java
index a2d891c2e6d..e326bacbaac 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DecimalColumnStatsDataInspector.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DecimalColumnStatsDataInspector.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.metastore.columnstats.cache;
import java.nio.ByteBuffer;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimatorFactory;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
@@ -27,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
public class DecimalColumnStatsDataInspector extends DecimalColumnStatsData {
private NumDistinctValueEstimator ndvEstimator;
+ private KllHistogramEstimator histogramEstimator;
public DecimalColumnStatsDataInspector() {
super();
@@ -41,6 +44,9 @@ public class DecimalColumnStatsDataInspector extends DecimalColumnStatsData {
if (other.ndvEstimator != null) {
super.setBitVectors(ndvEstimator.serialize());
}
+ if (other.histogramEstimator != null) {
+ super.setHistogram(histogramEstimator.serialize());
+ }
}
public DecimalColumnStatsDataInspector(DecimalColumnStatsData other) {
@@ -60,6 +66,14 @@ public class DecimalColumnStatsDataInspector extends DecimalColumnStatsData {
return super.getBitVectors();
}
+ @Override
+ public byte[] getHistogram() {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ return super.getHistogram();
+ }
+
@Override
public ByteBuffer bufferForBitVectors() {
if (ndvEstimator != null) {
@@ -68,6 +82,14 @@ public class DecimalColumnStatsDataInspector extends DecimalColumnStatsData {
return super.bufferForBitVectors();
}
+ @Override
+ public ByteBuffer bufferForHistogram() {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ return super.bufferForHistogram();
+ }
+
@Override
public void setBitVectors(byte[] bitVectors) {
super.setBitVectors(bitVectors);
@@ -80,12 +102,30 @@ public class DecimalColumnStatsDataInspector extends DecimalColumnStatsData {
this.ndvEstimator = null;
}
+ @Override
+ public void setHistogram(byte[] stats) {
+ super.setHistogram(stats);
+ this.histogramEstimator = null;
+ }
+
+ @Override
+ public void setHistogram(ByteBuffer stats) {
+ super.setHistogram(stats);
+ this.histogramEstimator = null;
+ }
+
@Override
public void unsetBitVectors() {
super.unsetBitVectors();
this.ndvEstimator = null;
}
+ @Override
+ public void unsetHistogram() {
+ super.unsetHistogram();
+ this.histogramEstimator = null;
+ }
+
@Override
public boolean isSetBitVectors() {
if (ndvEstimator != null) {
@@ -94,6 +134,14 @@ public class DecimalColumnStatsDataInspector extends DecimalColumnStatsData {
return super.isSetBitVectors();
}
+ @Override
+ public boolean isSetHistogram() {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ return super.isSetHistogram();
+ }
+
@Override
public void setBitVectorsIsSet(boolean value) {
if (ndvEstimator != null) {
@@ -102,6 +150,14 @@ public class DecimalColumnStatsDataInspector extends DecimalColumnStatsData {
super.setBitVectorsIsSet(value);
}
+ @Override
+ public void setHistogramIsSet(boolean value) {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ super.setHistogramIsSet(value);
+ }
+
public NumDistinctValueEstimator getNdvEstimator() {
if (ndvEstimator == null && isSetBitVectors() && getBitVectors().length != 0) {
updateNdvEstimator();
@@ -109,20 +165,42 @@ public class DecimalColumnStatsDataInspector extends DecimalColumnStatsData {
return ndvEstimator;
}
+ public KllHistogramEstimator getHistogramEstimator() {
+ if (histogramEstimator == null && isSetHistogram() && getHistogram().length != 0) {
+ updateHistogramEstimator();
+ }
+ return histogramEstimator;
+ }
+
public void setNdvEstimator(NumDistinctValueEstimator ndvEstimator) {
super.unsetBitVectors();
this.ndvEstimator = ndvEstimator;
}
+ public void setHistogramEstimator(KllHistogramEstimator histogramEstimator) {
+ super.unsetHistogram();
+ this.histogramEstimator = histogramEstimator;
+ }
+
private void updateBitVectors() {
super.setBitVectors(ndvEstimator.serialize());
this.ndvEstimator = null;
}
+ private void updateHistogram() {
+ super.setHistogram(histogramEstimator.serialize());
+ this.histogramEstimator = null;
+ }
+
private void updateNdvEstimator() {
this.ndvEstimator = NumDistinctValueEstimatorFactory
.getNumDistinctValueEstimator(super.getBitVectors());
super.unsetBitVectors();
}
+ private void updateHistogramEstimator() {
+ this.histogramEstimator = KllHistogramEstimatorFactory
+ .getKllHistogramEstimator(super.getHistogram());
+ super.unsetHistogram();
+ }
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DoubleColumnStatsDataInspector.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DoubleColumnStatsDataInspector.java
index 534116d9545..3a5f827ca71 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DoubleColumnStatsDataInspector.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DoubleColumnStatsDataInspector.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.metastore.columnstats.cache;
import java.nio.ByteBuffer;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimatorFactory;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
@@ -27,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
public class DoubleColumnStatsDataInspector extends DoubleColumnStatsData {
private NumDistinctValueEstimator ndvEstimator;
+ private KllHistogramEstimator histogramEstimator;
public DoubleColumnStatsDataInspector() {
super();
@@ -41,6 +44,9 @@ public class DoubleColumnStatsDataInspector extends DoubleColumnStatsData {
if (other.ndvEstimator != null) {
super.setBitVectors(ndvEstimator.serialize());
}
+ if (other.histogramEstimator != null) {
+ super.setHistogram(histogramEstimator.serialize());
+ }
}
public DoubleColumnStatsDataInspector(DoubleColumnStatsData other) {
@@ -60,6 +66,14 @@ public class DoubleColumnStatsDataInspector extends DoubleColumnStatsData {
return super.getBitVectors();
}
+ @Override
+ public byte[] getHistogram() {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ return super.getHistogram();
+ }
+
@Override
public ByteBuffer bufferForBitVectors() {
if (ndvEstimator != null) {
@@ -68,6 +82,14 @@ public class DoubleColumnStatsDataInspector extends DoubleColumnStatsData {
return super.bufferForBitVectors();
}
+ @Override
+ public ByteBuffer bufferForHistogram() {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ return super.bufferForHistogram();
+ }
+
@Override
public void setBitVectors(byte[] bitVectors) {
super.setBitVectors(bitVectors);
@@ -80,12 +102,30 @@ public class DoubleColumnStatsDataInspector extends DoubleColumnStatsData {
this.ndvEstimator = null;
}
+ @Override
+ public void setHistogram(byte[] stats) {
+ super.setHistogram(stats);
+ this.histogramEstimator = null;
+ }
+
+ @Override
+ public void setHistogram(ByteBuffer stats) {
+ super.setHistogram(stats);
+ this.histogramEstimator = null;
+ }
+
@Override
public void unsetBitVectors() {
super.unsetBitVectors();
this.ndvEstimator = null;
}
+ @Override
+ public void unsetHistogram() {
+ super.unsetHistogram();
+ this.histogramEstimator = null;
+ }
+
@Override
public boolean isSetBitVectors() {
if (ndvEstimator != null) {
@@ -94,6 +134,14 @@ public class DoubleColumnStatsDataInspector extends DoubleColumnStatsData {
return super.isSetBitVectors();
}
+ @Override
+ public boolean isSetHistogram() {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ return super.isSetHistogram();
+ }
+
@Override
public void setBitVectorsIsSet(boolean value) {
if (ndvEstimator != null) {
@@ -102,6 +150,14 @@ public class DoubleColumnStatsDataInspector extends DoubleColumnStatsData {
super.setBitVectorsIsSet(value);
}
+ @Override
+ public void setHistogramIsSet(boolean value) {
+ if (histogramEstimator != null) {
+ updateHistogram();
+ }
+ super.setHistogramIsSet(value);
+ }
+
public NumDistinctValueEstimator getNdvEstimator() {
if (ndvEstimator == null && isSetBitVectors() && getBitVectors().length != 0) {
updateNdvEstimator();
@@ -109,20 +165,42 @@ public class DoubleColumnStatsDataInspector extends DoubleColumnStatsData {
return ndvEstimator;
}
+ public KllHistogramEstimator getHistogramEstimator() {
+ if (histogramEstimator == null && isSetHistogram() && getHistogram().length != 0) {
+ updateHistogramEstimator();
+ }
+ return histogramEstimator;
+ }
+
public void setNdvEstimator(NumDistinctValueEstimator ndvEstimator) {
super.unsetBitVectors();
this.ndvEstimator = ndvEstimator;
}
+ public void setHistogramEstimator(KllHistogramEstimator histogramEstimator) {
+ super.unsetHistogram();
+ this.histogramEstimator = histogramEstimator;
+ }
+
private void updateBitVectors() {
super.setBitVectors(ndvEstimator.serialize());
this.ndvEstimator = null;
}
+ private void updateHistogram() {
+ super.setHistogram(histogramEstimator.serialize());
+ this.histogramEstimator = null;
+ }
+
private void updateNdvEstimator() {
this.ndvEstimator = NumDistinctValueEstimatorFactory
.getNumDistinctValueEstimator(super.getBitVectors());
super.unsetBitVectors();
}
+ private void updateHistogramEstimator() {
+ this.histogramEstimator = KllHistogramEstimatorFactory
+ .getKllHistogramEstimator(super.getHistogram());
+ super.unsetHistogram();
+ }
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/LongColumnStatsDataInspector.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/LongColumnStatsDataInspector.java
index f2470ec29a5..24a209126a3 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/LongColumnStatsDataInspector.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/LongColumnStatsDataInspector.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.metastore.columnstats.cache;
import java.nio.ByteBuffer;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimator;
+import org.apache.hadoop.hive.common.histogram.KllHistogramEstimatorFactory;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
@@ -27,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
public class LongColumnStatsDataInspector extends LongColumnStatsData {
private NumDistinctValueEstimator ndvEstimator;
+ private KllHistogramEstimator histogramEstimator;
public LongColumnStatsDataInspector() {
super();
@@ -41,6 +44,9 @@ public class LongColumnStatsDataInspector extends LongColumnStatsData {
if (other.ndvEstimator != null) {
super.setBitVectors(ndvEstimator.serialize());
}
+ if (other.histogramEstimator != null) {
+ super.setHistogram(histogramEstimator.serialize());
+ }
}
... 3845 lines suppressed ...