You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/08/04 21:14:26 UTC
[05/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
deleted file mode 100644
index e55c412..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implements
- IExtrapolatePartStatus {
-
- private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
-
- @Override
- public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
- List<ColumnStatistics> css) throws MetaException {
- ColumnStatisticsObj statsObj = null;
-
- // check if all the ColumnStatisticsObjs contain stats and all the ndv are
- // bitvectors
- boolean doAllPartitionContainStats = partNames.size() == css.size();
- LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
- NumDistinctValueEstimator ndvEstimator = null;
- String colType = null;
- for (ColumnStatistics cs : css) {
- if (cs.getStatsObjSize() != 1) {
- throw new MetaException(
- "The number of columns should be exactly one in aggrStats, but found "
- + cs.getStatsObjSize());
- }
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- if (statsObj == null) {
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
- .getStatsData().getSetField());
- }
- if (!cso.getStatsData().getDoubleStats().isSetBitVectors()
- || cso.getStatsData().getDoubleStats().getBitVectors().length() == 0) {
- ndvEstimator = null;
- break;
- } else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(cso.getStatsData().getDoubleStats().getBitVectors());
- if (ndvEstimator == null) {
- ndvEstimator = estimator;
- } else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
- break;
- }
- }
- }
- }
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- if (doAllPartitionContainStats || css.size() < 2) {
- DoubleColumnStatsData aggregateData = null;
- long lowerBound = 0;
- long higherBound = 0;
- double densityAvgSum = 0.0;
- for (ColumnStatistics cs : css) {
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
- lowerBound = Math.max(lowerBound, newData.getNumDVs());
- higherBound += newData.getNumDVs();
- densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
- if (ndvEstimator != null) {
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData
- .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- }
- }
- if (ndvEstimator != null) {
- // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
- // use uniform distribution assumption because we can merge bitvectors
- // to get a good estimation.
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- } else {
- long estimation;
- if (useDensityFunctionForNDVEstimation) {
- // We have estimation, lowerbound and higherbound. We use estimation
- // if it is between lowerbound and higherbound.
- double densityAvg = densityAvgSum / partNames.size();
- estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
- if (estimation < lowerBound) {
- estimation = lowerBound;
- } else if (estimation > higherBound) {
- estimation = higherBound;
- }
- } else {
- estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
- }
- aggregateData.setNumDVs(estimation);
- }
- columnStatisticsData.setDoubleStats(aggregateData);
- } else {
- // we need extrapolation
- LOG.debug("start extrapolation for " + colName);
- Map<String, Integer> indexMap = new HashMap<String, Integer>();
- for (int index = 0; index < partNames.size(); index++) {
- indexMap.put(partNames.get(index), index);
- }
- Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
- Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
- // while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
- double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
- // if not every partition uses bitvector for ndv, we just fall back to
- // the traditional extrapolation methods.
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
- }
- adjustedIndexMap.put(partName, (double) indexMap.get(partName));
- adjustedStatsMap.put(partName, cso.getStatsData());
- }
- } else {
- // we first merge all the adjacent bitvectors that we could merge and
- // derive new partition names and index.
- StringBuilder pseudoPartName = new StringBuilder();
- double pseudoIndexSum = 0;
- int length = 0;
- int curIndex = -1;
- DoubleColumnStatsData aggregateData = null;
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
- // newData.isSetBitVectors() should be true for sure because we
- // already checked it before.
- if (indexMap.get(partName) != curIndex) {
- // There is bitvector, but it is not adjacent to the previous ones.
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setDoubleStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
- }
- // reset everything
- pseudoPartName = new StringBuilder();
- pseudoIndexSum = 0;
- length = 0;
- ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- aggregateData = null;
- }
- curIndex = indexMap.get(partName);
- pseudoPartName.append(partName);
- pseudoIndexSum += curIndex;
- length++;
- curIndex++;
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
- newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setDoubleStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
- }
- }
- }
- extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
- adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
- }
- LOG.debug("Ndv estimatation for " + colName + " is "
- + columnStatisticsData.getDoubleStats().getNumDVs());
- statsObj.setStatsData(columnStatisticsData);
- return statsObj;
- }
-
- @Override
- public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
- int numPartsWithStats, Map<String, Double> adjustedIndexMap,
- Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- DoubleColumnStatsData extrapolateDoubleData = new DoubleColumnStatsData();
- Map<String, DoubleColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
- for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
- extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDoubleStats());
- }
- List<Map.Entry<String, DoubleColumnStatsData>> list = new LinkedList<Map.Entry<String, DoubleColumnStatsData>>(
- extractedAdjustedStatsMap.entrySet());
- // get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
- public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
- Map.Entry<String, DoubleColumnStatsData> o2) {
- return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
- }
- });
- double minInd = adjustedIndexMap.get(list.get(0).getKey());
- double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double lowValue = 0;
- double min = list.get(0).getValue().getLowValue();
- double max = list.get(list.size() - 1).getValue().getLowValue();
- if (minInd == maxInd) {
- lowValue = min;
- } else if (minInd < maxInd) {
- // left border is the min
- lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
- } else {
- // right border is the min
- lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
- }
-
- // get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
- public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
- Map.Entry<String, DoubleColumnStatsData> o2) {
- return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
- }
- });
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double highValue = 0;
- min = list.get(0).getValue().getHighValue();
- max = list.get(list.size() - 1).getValue().getHighValue();
- if (minInd == maxInd) {
- highValue = min;
- } else if (minInd < maxInd) {
- // right border is the max
- highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- highValue = (min + (max - min) * minInd / (minInd - maxInd));
- }
-
- // get the #nulls
- long numNulls = 0;
- for (Map.Entry<String, DoubleColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
- numNulls += entry.getValue().getNumNulls();
- }
- // we scale up sumNulls based on the number of partitions
- numNulls = numNulls * numParts / numPartsWithStats;
-
- // get the ndv
- long ndv = 0;
- long ndvMin = 0;
- long ndvMax = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
- public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
- Map.Entry<String, DoubleColumnStatsData> o2) {
- return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
- }
- });
- long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
- long higherBound = 0;
- for (Map.Entry<String, DoubleColumnStatsData> entry : list) {
- higherBound += entry.getValue().getNumDVs();
- }
- if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
- ndv = (long) ((highValue - lowValue) / densityAvg);
- if (ndv < lowerBound) {
- ndv = lowerBound;
- } else if (ndv > higherBound) {
- ndv = higherBound;
- }
- } else {
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- ndvMin = list.get(0).getValue().getNumDVs();
- ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
- if (minInd == maxInd) {
- ndv = ndvMin;
- } else if (minInd < maxInd) {
- // right border is the max
- ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
- }
- }
- extrapolateDoubleData.setLowValue(lowValue);
- extrapolateDoubleData.setHighValue(highValue);
- extrapolateDoubleData.setNumNulls(numNulls);
- extrapolateDoubleData.setNumDVs(ndv);
- extrapolateData.setDoubleStats(extrapolateDoubleData);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
deleted file mode 100644
index acf679e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Map;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-
-public interface IExtrapolatePartStatus {
- // The following function will extrapolate the stats when the column stats of
- // some partitions are missing.
- /**
- * @param extrapolateData
- * it will carry back the specific stats, e.g., DOUBLE_STATS or
- * LONG_STATS
- * @param numParts
- * the total number of partitions
- * @param numPartsWithStats
- * the number of partitions that have stats
- * @param adjustedIndexMap
- * the partition name to index map
- * @param adjustedStatsMap
- * the partition name to its stats map
- * @param densityAvg
- * the average of ndv density, which is useful when
- * useDensityFunctionForNDVEstimation is true.
- */
- public abstract void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
- int numPartsWithStats, Map<String, Double> adjustedIndexMap,
- Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg);
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
deleted file mode 100644
index 2ee09f3..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
- IExtrapolatePartStatus {
-
- private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
-
- @Override
- public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
- List<ColumnStatistics> css) throws MetaException {
- ColumnStatisticsObj statsObj = null;
-
- // check if all the ColumnStatisticsObjs contain stats and all the ndv are
- // bitvectors
- boolean doAllPartitionContainStats = partNames.size() == css.size();
- LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
- NumDistinctValueEstimator ndvEstimator = null;
- String colType = null;
- for (ColumnStatistics cs : css) {
- if (cs.getStatsObjSize() != 1) {
- throw new MetaException(
- "The number of columns should be exactly one in aggrStats, but found "
- + cs.getStatsObjSize());
- }
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- if (statsObj == null) {
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
- .getStatsData().getSetField());
- }
- if (!cso.getStatsData().getLongStats().isSetBitVectors()
- || cso.getStatsData().getLongStats().getBitVectors().length() == 0) {
- ndvEstimator = null;
- break;
- } else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(cso.getStatsData().getLongStats().getBitVectors());
- if (ndvEstimator == null) {
- ndvEstimator = estimator;
- } else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
- break;
- }
- }
- }
- }
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- if (doAllPartitionContainStats || css.size() < 2) {
- LongColumnStatsData aggregateData = null;
- long lowerBound = 0;
- long higherBound = 0;
- double densityAvgSum = 0.0;
- for (ColumnStatistics cs : css) {
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- LongColumnStatsData newData = cso.getStatsData().getLongStats();
- lowerBound = Math.max(lowerBound, newData.getNumDVs());
- higherBound += newData.getNumDVs();
- densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
- if (ndvEstimator != null) {
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData
- .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- }
- }
- if (ndvEstimator != null) {
- // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
- // use uniform distribution assumption because we can merge bitvectors
- // to get a good estimation.
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- } else {
- long estimation;
- if (useDensityFunctionForNDVEstimation) {
- // We have estimation, lowerbound and higherbound. We use estimation
- // if it is between lowerbound and higherbound.
- double densityAvg = densityAvgSum / partNames.size();
- estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
- if (estimation < lowerBound) {
- estimation = lowerBound;
- } else if (estimation > higherBound) {
- estimation = higherBound;
- }
- } else {
- estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
- }
- aggregateData.setNumDVs(estimation);
- }
- columnStatisticsData.setLongStats(aggregateData);
- } else {
- // we need extrapolation
- LOG.debug("start extrapolation for " + colName);
-
- Map<String, Integer> indexMap = new HashMap<String, Integer>();
- for (int index = 0; index < partNames.size(); index++) {
- indexMap.put(partNames.get(index), index);
- }
- Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
- Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
- // while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
- double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
- // if not every partition uses bitvector for ndv, we just fall back to
- // the traditional extrapolation methods.
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- LongColumnStatsData newData = cso.getStatsData().getLongStats();
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
- }
- adjustedIndexMap.put(partName, (double) indexMap.get(partName));
- adjustedStatsMap.put(partName, cso.getStatsData());
- }
- } else {
- // we first merge all the adjacent bitvectors that we could merge and
- // derive new partition names and index.
- StringBuilder pseudoPartName = new StringBuilder();
- double pseudoIndexSum = 0;
- int length = 0;
- int curIndex = -1;
- LongColumnStatsData aggregateData = null;
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- LongColumnStatsData newData = cso.getStatsData().getLongStats();
- // newData.isSetBitVectors() should be true for sure because we
- // already checked it before.
- if (indexMap.get(partName) != curIndex) {
- // There is bitvector, but it is not adjacent to the previous ones.
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setLongStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
- }
- // reset everything
- pseudoPartName = new StringBuilder();
- pseudoIndexSum = 0;
- length = 0;
- ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- aggregateData = null;
- }
- curIndex = indexMap.get(partName);
- pseudoPartName.append(partName);
- pseudoIndexSum += curIndex;
- length++;
- curIndex++;
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
- newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setLongStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
- }
- }
- }
- extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
- adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
- }
- statsObj.setStatsData(columnStatisticsData);
- LOG.debug("Ndv estimatation for " + colName + " is "
- + columnStatisticsData.getLongStats().getNumDVs());
- return statsObj;
- }
-
- @Override
- public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
- int numPartsWithStats, Map<String, Double> adjustedIndexMap,
- Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- LongColumnStatsData extrapolateLongData = new LongColumnStatsData();
- Map<String, LongColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
- for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
- extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getLongStats());
- }
- List<Map.Entry<String, LongColumnStatsData>> list = new LinkedList<Map.Entry<String, LongColumnStatsData>>(
- extractedAdjustedStatsMap.entrySet());
- // get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
- public int compare(Map.Entry<String, LongColumnStatsData> o1,
- Map.Entry<String, LongColumnStatsData> o2) {
- return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
- }
- });
- double minInd = adjustedIndexMap.get(list.get(0).getKey());
- double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long lowValue = 0;
- long min = list.get(0).getValue().getLowValue();
- long max = list.get(list.size() - 1).getValue().getLowValue();
- if (minInd == maxInd) {
- lowValue = min;
- } else if (minInd < maxInd) {
- // left border is the min
- lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
- } else {
- // right border is the min
- lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
- }
-
- // get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
- public int compare(Map.Entry<String, LongColumnStatsData> o1,
- Map.Entry<String, LongColumnStatsData> o2) {
- return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
- }
- });
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long highValue = 0;
- min = list.get(0).getValue().getHighValue();
- max = list.get(list.size() - 1).getValue().getHighValue();
- if (minInd == maxInd) {
- highValue = min;
- } else if (minInd < maxInd) {
- // right border is the max
- highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
- }
-
- // get the #nulls
- long numNulls = 0;
- for (Map.Entry<String, LongColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
- numNulls += entry.getValue().getNumNulls();
- }
- // we scale up sumNulls based on the number of partitions
- numNulls = numNulls * numParts / numPartsWithStats;
-
- // get the ndv
- long ndv = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
- public int compare(Map.Entry<String, LongColumnStatsData> o1,
- Map.Entry<String, LongColumnStatsData> o2) {
- return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
- }
- });
- long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
- long higherBound = 0;
- for (Map.Entry<String, LongColumnStatsData> entry : list) {
- higherBound += entry.getValue().getNumDVs();
- }
- if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
- ndv = (long) ((highValue - lowValue) / densityAvg);
- if (ndv < lowerBound) {
- ndv = lowerBound;
- } else if (ndv > higherBound) {
- ndv = higherBound;
- }
- } else {
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- min = list.get(0).getValue().getNumDVs();
- max = list.get(list.size() - 1).getValue().getNumDVs();
- if (minInd == maxInd) {
- ndv = min;
- } else if (minInd < maxInd) {
- // right border is the max
- ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
- }
- }
- extrapolateLongData.setLowValue(lowValue);
- extrapolateLongData.setHighValue(highValue);
- extrapolateLongData.setNumNulls(numNulls);
- extrapolateLongData.setNumDVs(ndv);
- extrapolateData.setLongStats(extrapolateLongData);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java
deleted file mode 100644
index 4c2d1bc..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-
-public class BinaryColumnStatsMerger extends ColumnStatsMerger {
-
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats();
- BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats();
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java
deleted file mode 100644
index 8e50153..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-
-public class BooleanColumnStatsMerger extends ColumnStatsMerger {
-
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- BooleanColumnStatsData aggregateData = aggregateColStats.getStatsData().getBooleanStats();
- BooleanColumnStatsData newData = newColStats.getStatsData().getBooleanStats();
- aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
- aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java
deleted file mode 100644
index 474d4dd..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class ColumnStatsMerger {
- protected final Logger LOG = LoggerFactory.getLogger(ColumnStatsMerger.class.getName());
-
- public abstract void merge(ColumnStatisticsObj aggregateColStats,
- ColumnStatisticsObj newColStats);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java
deleted file mode 100644
index 0ce1847..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-
-public class ColumnStatsMergerFactory {
-
- private ColumnStatsMergerFactory() {
- }
-
- public static ColumnStatsMerger getColumnStatsMerger(ColumnStatisticsObj statsObjNew,
- ColumnStatisticsObj statsObjOld) {
- ColumnStatsMerger agg;
- _Fields typeNew = statsObjNew.getStatsData().getSetField();
- _Fields typeOld = statsObjOld.getStatsData().getSetField();
- // make sure that they have the same type
- typeNew = typeNew == typeOld ? typeNew : null;
- switch (typeNew) {
- case BOOLEAN_STATS:
- agg = new BooleanColumnStatsMerger();
- break;
- case LONG_STATS: {
- agg = new LongColumnStatsMerger();
- break;
- }
- case DOUBLE_STATS: {
- agg = new DoubleColumnStatsMerger();
- break;
- }
- case STRING_STATS: {
- agg = new StringColumnStatsMerger();
- break;
- }
- case BINARY_STATS:
- agg = new BinaryColumnStatsMerger();
- break;
- case DECIMAL_STATS: {
- agg = new DecimalColumnStatsMerger();
- break;
- }
- case DATE_STATS: {
- agg = new DateColumnStatsMerger();
- break;
- }
- default:
- throw new IllegalArgumentException("Unknown stats type " + typeNew.toString());
- }
- return agg;
- }
-
- public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
- ColumnStatisticsObj cso = new ColumnStatisticsObj();
- ColumnStatisticsData csd = new ColumnStatisticsData();
- cso.setColName(colName);
- cso.setColType(colType);
- switch (type) {
- case BOOLEAN_STATS:
- csd.setBooleanStats(new BooleanColumnStatsData());
- break;
-
- case LONG_STATS:
- csd.setLongStats(new LongColumnStatsData());
- break;
-
- case DOUBLE_STATS:
- csd.setDoubleStats(new DoubleColumnStatsData());
- break;
-
- case STRING_STATS:
- csd.setStringStats(new StringColumnStatsData());
- break;
-
- case BINARY_STATS:
- csd.setBinaryStats(new BinaryColumnStatsData());
- break;
-
- case DECIMAL_STATS:
- csd.setDecimalStats(new DecimalColumnStatsData());
- break;
-
- case DATE_STATS:
- csd.setDateStats(new DateColumnStatsData());
- break;
-
- default:
- throw new IllegalArgumentException("Unknown stats type");
- }
-
- cso.setStatsData(csd);
- return cso;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java
deleted file mode 100644
index 2542a00..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Date;
-import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
-
-public class DateColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DateColumnStatsData aggregateData = aggregateColStats.getStatsData().getDateStats();
- DateColumnStatsData newData = newColStats.getStatsData().getDateStats();
- Date lowValue = aggregateData.getLowValue().compareTo(newData.getLowValue()) < 0 ? aggregateData
- .getLowValue() : newData.getLowValue();
- aggregateData.setLowValue(lowValue);
- Date highValue = aggregateData.getHighValue().compareTo(newData.getHighValue()) >= 0 ? aggregateData
- .getHighValue() : newData.getHighValue();
- aggregateData.setHighValue(highValue);
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java
deleted file mode 100644
index 4e8e129..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-
-public class DecimalColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DecimalColumnStatsData aggregateData = aggregateColStats.getStatsData().getDecimalStats();
- DecimalColumnStatsData newData = newColStats.getStatsData().getDecimalStats();
- Decimal lowValue = aggregateData.getLowValue() != null
- && (aggregateData.getLowValue().compareTo(newData.getLowValue()) > 0) ? aggregateData
- .getLowValue() : newData.getLowValue();
- aggregateData.setLowValue(lowValue);
- Decimal highValue = aggregateData.getHighValue() != null
- && (aggregateData.getHighValue().compareTo(newData.getHighValue()) > 0) ? aggregateData
- .getHighValue() : newData.getHighValue();
- aggregateData.setHighValue(highValue);
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java
deleted file mode 100644
index 4ef5c39..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-
-public class DoubleColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DoubleColumnStatsData aggregateData = aggregateColStats.getStatsData().getDoubleStats();
- DoubleColumnStatsData newData = newColStats.getStatsData().getDoubleStats();
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java
deleted file mode 100644
index acf7f03..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-
-public class LongColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- LongColumnStatsData aggregateData = aggregateColStats.getStatsData().getLongStats();
- LongColumnStatsData newData = newColStats.getStatsData().getLongStats();
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java
deleted file mode 100644
index b3cd33c..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-
-public class StringColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- StringColumnStatsData aggregateData = aggregateColStats.getStatsData().getStringStats();
- StringColumnStatsData newData = newColStats.getStatsData().getStringStats();
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
----------------------------------------------------------------------
diff --git a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto b/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
deleted file mode 100644
index 53c381b..0000000
--- a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
+++ /dev/null
@@ -1,368 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-enum PrincipalType {
- USER = 0;
- ROLE = 1;
-}
-
-message AggrStats {
- required int64 parts_found = 1;
- repeated ColumnStats col_stats = 2;
-}
-
-message AggrStatsBloomFilter {
- message BloomFilter {
- required int32 num_bits = 1;
- required int32 num_funcs = 2;
- repeated int64 bits = 3;
- }
- required bytes db_name = 1;
- required bytes table_name = 2;
- required BloomFilter bloom_filter = 3;
- required int64 aggregated_at = 4;
-}
-
-message AggrStatsInvalidatorFilter {
- message Entry {
- required bytes db_name = 1;
- required bytes table_name = 2;
- required bytes part_name = 3;
- }
-
- repeated Entry to_invalidate = 1;
- required int64 run_every = 2;
- required int64 max_cache_entry_life = 3;
-}
-
-message ColumnStats {
-
- message BooleanStats {
- optional int64 num_trues = 1;
- optional int64 num_falses = 2;
- }
-
- message LongStats {
- optional sint64 low_value = 1;
- optional sint64 high_value = 2;
- }
-
- message DoubleStats {
- optional double low_value = 1;
- optional double high_value = 2;
- }
-
- message StringStats {
- optional int64 max_col_length = 1;
- optional double avg_col_length = 2;
- }
-
- message DecimalStats {
- message Decimal {
- required bytes unscaled = 1;
- required int32 scale = 2;
- }
- optional Decimal low_value = 1;
- optional Decimal high_value = 2;
- }
-
- optional int64 last_analyzed = 1;
- required string column_type = 2;
- optional int64 num_nulls = 3;
- optional int64 num_distinct_values = 4;
- optional BooleanStats bool_stats = 5;
- optional LongStats long_stats = 6;
- optional DoubleStats double_stats = 7;
- optional StringStats string_stats = 8;
- optional StringStats binary_stats = 9;
- optional DecimalStats decimal_stats = 10;
- optional string column_name = 11;
- optional string bit_vectors = 12;
-}
-
-message Database {
- optional string description = 1;
- optional string uri = 2;
- optional Parameters parameters = 3;
- optional PrincipalPrivilegeSet privileges = 4;
- optional string owner_name = 5;
- optional PrincipalType owner_type = 6;
-}
-
-message DelegationToken {
- required string token_str = 1;
-}
-
-message FieldSchema {
- required string name = 1;
- required string type = 2;
- optional string comment = 3;
-}
-
-message Function {
- enum FunctionType {
- JAVA = 1;
- }
-
- message ResourceUri {
- enum ResourceType {
- JAR = 1;
- FILE = 2;
- ARCHIVE = 3;
- }
- required ResourceType resource_type = 1;
- required string uri = 2;
- }
-
- optional string class_name = 1;
- optional string owner_name = 2;
- optional PrincipalType owner_type = 3;
- optional sint64 create_time = 4;
- optional FunctionType function_type = 5;
- repeated ResourceUri resource_uris = 6;
-}
-
-message MasterKey {
- required string master_key = 1;
-}
-
-message ParameterEntry {
- required string key = 1;
- required string value = 2;
-}
-
-message Parameters {
- repeated ParameterEntry parameter = 1;
-}
-
-message Partition {
- optional int64 create_time = 1;
- optional int64 last_access_time = 2;
- optional string location = 3;
- optional Parameters sd_parameters = 4; // storage descriptor parameters
- required bytes sd_hash = 5;
- optional Parameters parameters = 6; // partition parameters
- // We don't support partition level privileges
-}
-
-message PrincipalPrivilegeSetEntry {
- required string principal_name = 1;
- repeated PrivilegeGrantInfo privileges = 2;
-}
-
-message PrincipalPrivilegeSet {
- repeated PrincipalPrivilegeSetEntry users = 1;
- repeated PrincipalPrivilegeSetEntry roles = 2;
-}
-
-message PrivilegeGrantInfo {
- optional string privilege = 1;
- optional int64 create_time = 2;
- optional string grantor = 3;
- optional PrincipalType grantor_type = 4;
- optional bool grant_option = 5;
-}
-
-message RoleGrantInfo {
- required string principal_name = 1;
- required PrincipalType principal_type = 2;
- optional int64 add_time = 3;
- optional string grantor = 4;
- optional PrincipalType grantor_type = 5;
- optional bool grant_option = 6;
-}
-
-message RoleGrantInfoList {
- repeated RoleGrantInfo grant_info = 1;
-}
-
-message RoleList {
- repeated string role = 1;
-}
-
-message Role {
- optional int64 create_time = 1;
- optional string owner_name = 2;
-}
-
-message StorageDescriptor {
- message Order {
- required string column_name = 1;
- optional sint32 order = 2 [default = 1];
- }
-
- message SerDeInfo {
- optional string name = 1;
- optional string serialization_lib = 2;
- optional Parameters parameters = 3;
- }
-
- message SkewedInfo {
- message SkewedColValueList {
- repeated string skewed_col_value = 1;
- }
-
- message SkewedColValueLocationMap {
- repeated string key = 1;
- required string value = 2;
- }
-
- repeated string skewed_col_names = 1;
- repeated SkewedColValueList skewed_col_values = 2;
- repeated SkewedColValueLocationMap skewed_col_value_location_maps = 3;
- }
-
- repeated FieldSchema cols = 1;
- optional string input_format = 2;
- optional string output_format = 3;
- optional bool is_compressed = 4;
- optional sint32 num_buckets = 5;
- optional SerDeInfo serde_info = 6;
- repeated string bucket_cols = 7;
- repeated Order sort_cols = 8;
- optional SkewedInfo skewed_info = 9;
- optional bool stored_as_sub_directories = 10;
-}
-
-message Table {
- optional string owner = 1;
- optional int64 create_time = 2;
- optional int64 last_access_time = 3;
- optional int64 retention = 4;
- optional string location = 5;
- optional Parameters sd_parameters = 6; // storage descriptor parameters
- required bytes sd_hash = 7;
- repeated FieldSchema partition_keys = 8;
- optional Parameters parameters = 9;
- optional string view_original_text = 10;
- optional string view_expanded_text = 11;
- optional string table_type = 12;
- optional PrincipalPrivilegeSet privileges = 13;
- optional bool is_temporary = 14;
- optional bool is_rewrite_enabled = 15;
-}
-
-message Index {
- optional string indexHandlerClass = 1; // reserved
- required string dbName = 2;
- required string origTableName = 3;
- optional string location = 4;
- optional Parameters sd_parameters = 5; // storage descriptor parameters
- optional int32 createTime = 6;
- optional int32 lastAccessTime = 7;
- optional string indexTableName = 8;
- optional bytes sd_hash = 9;
- optional Parameters parameters = 10;
- optional bool deferredRebuild = 11;
-}
-
-message PartitionKeyComparator {
- required string names = 1;
- required string types = 2;
- message Mark {
- required string value = 1;
- required bool inclusive = 2;
- }
- message Range {
- required string key = 1;
- optional Mark start = 2;
- optional Mark end = 3;
- }
- message Operator {
- enum Type {
- LIKE = 0;
- NOTEQUALS = 1;
- }
- required Type type = 1;
- required string key = 2;
- required string val = 3;
- }
- repeated Operator op = 3;
- repeated Range range = 4;
-}
-
-message PrimaryKey {
- message PrimaryKeyColumn {
- required string column_name = 1;
- required sint32 key_seq = 2;
- }
-
- required string pk_name = 1;
- repeated PrimaryKeyColumn cols = 2;
- optional bool enable_constraint = 3;
- optional bool validate_constraint = 4;
- optional bool rely_constraint = 5;
-}
-
-message ForeignKeys {
- message ForeignKey {
- message ForeignKeyColumn {
- required string column_name = 1;
- required string referenced_column_name = 2;
- required sint32 key_seq = 3;
- }
-
- required string fk_name = 1;
- required string referenced_db_name = 2;
- required string referenced_table_name = 3;
- optional string referenced_pk_name = 4;
- optional int32 update_rule = 5;
- optional int32 delete_rule = 6;
- repeated ForeignKeyColumn cols = 7;
- optional bool enable_constraint = 8;
- optional bool validate_constraint = 9;
- optional bool rely_constraint = 10;
- }
-
- repeated ForeignKey fks = 1;
-}
-
-message UniqueConstraints {
- message UniqueConstraint {
- message UniqueConstraintColumn {
- required string column_name = 1;
- required sint32 key_seq = 2;
- }
-
- required string uk_name = 1;
- repeated UniqueConstraintColumn cols = 2;
- optional bool enable_constraint = 3;
- optional bool validate_constraint = 4;
- optional bool rely_constraint = 5;
- }
-
- repeated UniqueConstraint uks = 1;
-}
-
-message NotNullConstraints {
- message NotNullConstraint {
- message NotNullConstraintColumn {
- required string column_name = 1;
- }
-
- required string nn_name = 1;
- repeated NotNullConstraintColumn cols = 2;
- optional bool enable_constraint = 3;
- optional bool validate_constraint = 4;
- optional bool rely_constraint = 5;
- }
-
- repeated NotNullConstraint nns = 1;
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
deleted file mode 100644
index 784648a..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.FileFormatProxy;
-import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-
-/**
- * Mock utilities for HBaseStore testing
- */
-public class MockUtils {
-
- /**
- * The default impl is in ql package and is not available in unit tests.
- */
- public static class NOOPProxy implements PartitionExpressionProxy {
-
- @Override
- public String convertExprToFilter(byte[] expr) throws MetaException {
- return null;
- }
-
- @Override
- public boolean filterPartitionsByExpr(List<String> partColumnNames,
- List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr, String defaultPartitionName,
- List<String> partitionNames) throws MetaException {
- return false;
- }
-
- @Override
- public SearchArgument createSarg(byte[] expr) {
- return null;
- }
-
- @Override
- public FileMetadataExprType getMetadataType(String inputFormat) {
- return null;
- }
-
- @Override
- public FileFormatProxy getFileFormatProxy(FileMetadataExprType type) {
- return null;
- }
- }
-
- static HBaseStore init(Configuration conf, HTableInterface htable,
- final SortedMap<String, Cell> rows) throws IOException {
- ((HiveConf)conf).setVar(ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, NOOPProxy.class.getName());
- Mockito.when(htable.get(Mockito.any(Get.class))).thenAnswer(new Answer<Result>() {
- @Override
- public Result answer(InvocationOnMock invocation) throws Throwable {
- Get get = (Get) invocation.getArguments()[0];
- Cell cell = rows.get(new String(get.getRow()));
- if (cell == null) {
- return new Result();
- } else {
- return Result.create(new Cell[]{cell});
- }
- }
- });
-
- Mockito.when(htable.get(Mockito.anyListOf(Get.class))).thenAnswer(new Answer<Result[]>() {
- @Override
- public Result[] answer(InvocationOnMock invocation) throws Throwable {
- @SuppressWarnings("unchecked")
- List<Get> gets = (List<Get>) invocation.getArguments()[0];
- Result[] results = new Result[gets.size()];
- for (int i = 0; i < gets.size(); i++) {
- Cell cell = rows.get(new String(gets.get(i).getRow()));
- Result result;
- if (cell == null) {
- result = new Result();
- } else {
- result = Result.create(new Cell[]{cell});
- }
- results[i] = result;
- }
- return results;
- }
- });
-
- Mockito.when(htable.getScanner(Mockito.any(Scan.class))).thenAnswer(new Answer<ResultScanner>() {
- @Override
- public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
- Scan scan = (Scan)invocation.getArguments()[0];
- List<Result> results = new ArrayList<Result>();
- String start = new String(scan.getStartRow());
- String stop = new String(scan.getStopRow());
- SortedMap<String, Cell> sub = rows.subMap(start, stop);
- for (Map.Entry<String, Cell> e : sub.entrySet()) {
- results.add(Result.create(new Cell[]{e.getValue()}));
- }
-
- final Iterator<Result> iter = results.iterator();
-
- return new ResultScanner() {
- @Override
- public Result next() throws IOException {
- return null;
- }
-
- @Override
- public Result[] next(int nbRows) throws IOException {
- return new Result[0];
- }
-
- @Override
- public void close() {
-
- }
-
- @Override
- public Iterator<Result> iterator() {
- return iter;
- }
- };
- }
- });
-
- Mockito.doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- Put put = (Put)invocation.getArguments()[0];
- rows.put(new String(put.getRow()), put.getFamilyCellMap().firstEntry().getValue().get(0));
- return null;
- }
- }).when(htable).put(Mockito.any(Put.class));
-
- Mockito.when(htable.checkAndPut(Mockito.any(byte[].class), Mockito.any(byte[].class),
- Mockito.any(byte[].class), Mockito.any(byte[].class), Mockito.any(Put.class))).thenAnswer(
- new Answer<Boolean>() {
-
- @Override
- public Boolean answer(InvocationOnMock invocation) throws Throwable {
- // Always say it succeeded and overwrite
- Put put = (Put)invocation.getArguments()[4];
- rows.put(new String(put.getRow()),
- put.getFamilyCellMap().firstEntry().getValue().get(0));
- return true;
- }
- });
-
- Mockito.doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- Delete del = (Delete)invocation.getArguments()[0];
- rows.remove(new String(del.getRow()));
- return null;
- }
- }).when(htable).delete(Mockito.any(Delete.class));
-
- Mockito.when(htable.checkAndDelete(Mockito.any(byte[].class), Mockito.any(byte[].class),
- Mockito.any(byte[].class), Mockito.any(byte[].class), Mockito.any(Delete.class))).thenAnswer(
- new Answer<Boolean>() {
-
- @Override
- public Boolean answer(InvocationOnMock invocation) throws Throwable {
- // Always say it succeeded
- Delete del = (Delete)invocation.getArguments()[4];
- rows.remove(new String(del.getRow()));
- return true;
- }
- });
-
- // Mock connection
- HBaseConnection hconn = Mockito.mock(HBaseConnection.class);
- Mockito.when(hconn.getHBaseTable(Mockito.anyString())).thenReturn(htable);
- HiveConf.setVar(conf, HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS, HBaseReadWrite.TEST_CONN);
- HBaseReadWrite.setTestConnection(hconn);
- HBaseReadWrite.setConf(conf);
- HBaseStore store = new HBaseStore();
- store.setConf(conf);
- return store;
- }
-}