You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/08/04 21:14:22 UTC

[01/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Repository: hive
Updated Branches:
  refs/heads/master c9c716246 -> 5e0615573


http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreBitVector.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreBitVector.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreBitVector.java
deleted file mode 100644
index b1dc542..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreBitVector.java
+++ /dev/null
@@ -1,622 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class TestHBaseStoreBitVector {
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreBitVector.class.getName());
-  static Map<String, String> emptyParameters = new HashMap<String, String>();
-  // Table with NUM_PART_KEYS partitioning keys and NUM_PARTITIONS values per key
-  static final int NUM_PART_KEYS = 1;
-  static final int NUM_PARTITIONS = 5;
-  static final String DB = "db";
-  static final String TBL = "tbl";
-  static final String COL = "col";
-  static final String PART_KEY_PREFIX = "part";
-  static final String PART_VAL_PREFIX = "val";
-  static final String PART_KV_SEPARATOR = "=";
-  static final List<String> PART_KEYS = new ArrayList<String>();
-  static final List<String> PART_VALS = new ArrayList<String>();
-  // Initialize mock partitions
-  static {
-    for (int i = 1; i <= NUM_PART_KEYS; i++) {
-      PART_KEYS.add(PART_KEY_PREFIX + i);
-    }
-    for (int i = 1; i <= NUM_PARTITIONS; i++) {
-      PART_VALS.add(PART_VAL_PREFIX + i);
-    }
-  }
-  static final long DEFAULT_TIME = System.currentTimeMillis();
-  static final String PART_KEY = "part";
-  static final String LONG_COL = "longCol";
-  static final String LONG_TYPE = "long";
-  static final String INT_TYPE = "int";
-  static final String INT_VAL = "1234";
-  static final String DOUBLE_COL = "doubleCol";
-  static final String DOUBLE_TYPE = "double";
-  static final String DOUBLE_VAL = "3.1415";
-  static final String STRING_COL = "stringCol";
-  static final String STRING_TYPE = "string";
-  static final String STRING_VAL = "stringval";
-  static final String DECIMAL_COL = "decimalCol";
-  static final String DECIMAL_TYPE = "decimal(5,3)";
-  static final String DECIMAL_VAL = "12.123";
-  static List<ColumnStatisticsObj> longColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-  static List<ColumnStatisticsObj> doubleColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-  static List<ColumnStatisticsObj> stringColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-  static List<ColumnStatisticsObj> decimalColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-
-  @Rule public ExpectedException thrown = ExpectedException.none();
-  @Mock HTableInterface htable;
-  SortedMap<String, Cell> rows = new TreeMap<>();
-  HBaseStore store;
-
-
-  @BeforeClass
-  public static void beforeTest() {
-    // All data intitializations
-    populateMockStats();
-  }
-
-  private static void populateMockStats() {
-    ColumnStatisticsObj statsObj;
-    // Add NUM_PARTITIONS ColumnStatisticsObj of each type
-    // For aggregate stats test, we'll treat each ColumnStatisticsObj as stats for 1 partition
-    // For the rest, we'll just pick the 1st ColumnStatisticsObj from this list and use it
-    for (int i = 0; i < NUM_PARTITIONS; i++) {
-      statsObj = mockLongStats(i);
-      longColStatsObjs.add(statsObj);
-      statsObj = mockDoubleStats(i);
-      doubleColStatsObjs.add(statsObj);
-      statsObj = mockStringStats(i);
-      stringColStatsObjs.add(statsObj);
-      statsObj = mockDecimalStats(i);
-      decimalColStatsObjs.add(statsObj);
-    }
-  }
-
-  private static ColumnStatisticsObj mockLongStats(int i) {
-    long high = 120938479124L + 100*i;
-    long low = -12341243213412124L - 50*i;
-    long nulls = 23 + i;
-    long dVs = 213L + 10*i;
-    String bitVectors = "{0, 1, 2, 3, 4, 5, 6, 7, 8}{1, 2, 3, 4, 5, 6, 7, 8}";
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(LONG_COL);
-    colStatsObj.setColType(LONG_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    LongColumnStatsData longData = new LongColumnStatsData();
-    longData.setHighValue(high);
-    longData.setLowValue(low);
-    longData.setNumNulls(nulls);
-    longData.setNumDVs(dVs);
-    longData.setBitVectors(bitVectors);
-    data.setLongStats(longData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  private static ColumnStatisticsObj mockDoubleStats(int i) {
-    double high = 123423.23423 + 100*i;
-    double low = 0.00001234233 - 50*i;
-    long nulls = 92 + i;
-    long dVs = 1234123421L + 10*i;
-    String bitVectors = "{0, 1, 2, 3, 4, 5, 6, 7, 8}{0, 2, 3, 4, 5, 6, 7, 8}";
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(DOUBLE_COL);
-    colStatsObj.setColType(DOUBLE_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    DoubleColumnStatsData doubleData = new DoubleColumnStatsData();
-    doubleData.setHighValue(high);
-    doubleData.setLowValue(low);
-    doubleData.setNumNulls(nulls);
-    doubleData.setNumDVs(dVs);
-    doubleData.setBitVectors(bitVectors);
-    data.setDoubleStats(doubleData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  private static ColumnStatisticsObj mockStringStats(int i) {
-    long maxLen = 1234 + 10*i;
-    double avgLen = 32.3 + i;
-    long nulls = 987 + 10*i;
-    long dVs = 906 + i;
-    String bitVectors = "{0, 1, 2, 3, 4, 5, 6, 7, 8}{0, 1, 3, 4, 5, 6, 7, 8}";
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(STRING_COL);
-    colStatsObj.setColType(STRING_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    StringColumnStatsData stringData = new StringColumnStatsData();
-    stringData.setMaxColLen(maxLen);
-    stringData.setAvgColLen(avgLen);
-    stringData.setNumNulls(nulls);
-    stringData.setNumDVs(dVs);
-    stringData.setBitVectors(bitVectors);
-    data.setStringStats(stringData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  private static ColumnStatisticsObj mockDecimalStats(int i) {
-    Decimal high = new Decimal();
-    high.setScale((short)3);
-    String strHigh = String.valueOf(3876 + 100*i);
-    high.setUnscaled(strHigh.getBytes());
-    Decimal low = new Decimal();
-    low.setScale((short)3);
-    String strLow = String.valueOf(38 + i);
-    low.setUnscaled(strLow.getBytes());
-    long nulls = 13 + i;
-    long dVs = 923947293L + 100*i;
-    String bitVectors = "{0, 1, 2, 3, 4, 5, 6, 7, 8}{0, 1, 2, 4, 5, 6, 7, 8}";
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(DECIMAL_COL);
-    colStatsObj.setColType(DECIMAL_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    DecimalColumnStatsData decimalData = new DecimalColumnStatsData();
-    decimalData.setHighValue(high);
-    decimalData.setLowValue(low);
-    decimalData.setNumNulls(nulls);
-    decimalData.setNumDVs(dVs);
-    decimalData.setBitVectors(bitVectors);
-    data.setDecimalStats(decimalData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  @AfterClass
-  public static void afterTest() {
-  }
-
-
-  @Before
-  public void init() throws IOException {
-    MockitoAnnotations.initMocks(this);
-    HiveConf conf = new HiveConf();
-    conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
-    store = MockUtils.init(conf, htable, rows);
-  }
-
-  @Test
-  public void longTableStatistics() throws Exception {
-    createMockTable(LONG_COL, LONG_TYPE);
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = longColStatsObjs.get(0);
-    LongColumnStatsData longData = obj.getStatsData().getLongStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(LONG_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, dataFromDB.getSetField());
-    // Compare LongColumnStatsData
-    LongColumnStatsData longDataFromDB = dataFromDB.getLongStats();
-    Assert.assertEquals(longData.getHighValue(), longDataFromDB.getHighValue());
-    Assert.assertEquals(longData.getLowValue(), longDataFromDB.getLowValue());
-    Assert.assertEquals(longData.getNumNulls(), longDataFromDB.getNumNulls());
-    Assert.assertEquals(longData.getNumDVs(), longDataFromDB.getNumDVs());
-    Assert.assertEquals(longData.getBitVectors(), longDataFromDB.getBitVectors());
-  }
-
-  @Test
-  public void doubleTableStatistics() throws Exception {
-    createMockTable(DOUBLE_COL, DOUBLE_TYPE);
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = doubleColStatsObjs.get(0);
-    DoubleColumnStatsData doubleData = obj.getStatsData().getDoubleStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(DOUBLE_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, dataFromDB.getSetField());
-    // Compare DoubleColumnStatsData
-    DoubleColumnStatsData doubleDataFromDB = dataFromDB.getDoubleStats();
-    Assert.assertEquals(doubleData.getHighValue(), doubleDataFromDB.getHighValue(), 0.01);
-    Assert.assertEquals(doubleData.getLowValue(), doubleDataFromDB.getLowValue(), 0.01);
-    Assert.assertEquals(doubleData.getNumNulls(), doubleDataFromDB.getNumNulls());
-    Assert.assertEquals(doubleData.getNumDVs(), doubleDataFromDB.getNumDVs());
-    Assert.assertEquals(doubleData.getBitVectors(), doubleDataFromDB.getBitVectors());
-  }
-
-  @Test
-  public void stringTableStatistics() throws Exception {
-    createMockTable(STRING_COL, STRING_TYPE);
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = stringColStatsObjs.get(0);
-    StringColumnStatsData stringData = obj.getStatsData().getStringStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(STRING_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, dataFromDB.getSetField());
-    // Compare StringColumnStatsData
-    StringColumnStatsData stringDataFromDB = dataFromDB.getStringStats();
-    Assert.assertEquals(stringData.getMaxColLen(), stringDataFromDB.getMaxColLen());
-    Assert.assertEquals(stringData.getAvgColLen(), stringDataFromDB.getAvgColLen(), 0.01);
-    Assert.assertEquals(stringData.getNumNulls(), stringDataFromDB.getNumNulls());
-    Assert.assertEquals(stringData.getNumDVs(), stringDataFromDB.getNumDVs());
-    Assert.assertEquals(stringData.getBitVectors(), stringDataFromDB.getBitVectors());
-  }
-
-  @Test
-  public void decimalTableStatistics() throws Exception {
-    createMockTable(DECIMAL_COL, DECIMAL_TYPE);
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = decimalColStatsObjs.get(0);
-    DecimalColumnStatsData decimalData = obj.getStatsData().getDecimalStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(DECIMAL_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, dataFromDB.getSetField());
-    // Compare DecimalColumnStatsData
-    DecimalColumnStatsData decimalDataFromDB = dataFromDB.getDecimalStats();
-    Assert.assertEquals(decimalData.getHighValue(), decimalDataFromDB.getHighValue());
-    Assert.assertEquals(decimalData.getLowValue(), decimalDataFromDB.getLowValue());
-    Assert.assertEquals(decimalData.getNumNulls(), decimalDataFromDB.getNumNulls());
-    Assert.assertEquals(decimalData.getNumDVs(), decimalDataFromDB.getNumDVs());
-    Assert.assertEquals(decimalData.getBitVectors(), decimalDataFromDB.getBitVectors());
-  }
-
-  @Test
-  public void longPartitionStatistics() throws Exception {
-    createMockTableAndPartition(INT_TYPE, INT_VAL);
-    // Add partition stats for: LONG_COL and partition: {PART_KEY, INT_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, INT_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = longColStatsObjs.get(0);
-    LongColumnStatsData longData = obj.getStatsData().getLongStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(INT_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, dataFromDB.getSetField());
-    // Compare LongColumnStatsData
-    LongColumnStatsData longDataFromDB = dataFromDB.getLongStats();
-    Assert.assertEquals(longData.getHighValue(), longDataFromDB.getHighValue());
-    Assert.assertEquals(longData.getLowValue(), longDataFromDB.getLowValue());
-    Assert.assertEquals(longData.getNumNulls(), longDataFromDB.getNumNulls());
-    Assert.assertEquals(longData.getNumDVs(), longDataFromDB.getNumDVs());
-    Assert.assertEquals(longData.getBitVectors(), longDataFromDB.getBitVectors());
-  }
-
-  @Test
-  public void doublePartitionStatistics() throws Exception {
-    createMockTableAndPartition(DOUBLE_TYPE, DOUBLE_VAL);
-    // Add partition stats for: DOUBLE_COL and partition: {PART_KEY, DOUBLE_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, DOUBLE_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = doubleColStatsObjs.get(0);
-    DoubleColumnStatsData doubleData = obj.getStatsData().getDoubleStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(DOUBLE_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, dataFromDB.getSetField());
-    // Compare DoubleColumnStatsData
-    DoubleColumnStatsData doubleDataFromDB = dataFromDB.getDoubleStats();
-    Assert.assertEquals(doubleData.getHighValue(), doubleDataFromDB.getHighValue(), 0.01);
-    Assert.assertEquals(doubleData.getLowValue(), doubleDataFromDB.getLowValue(), 0.01);
-    Assert.assertEquals(doubleData.getNumNulls(), doubleDataFromDB.getNumNulls());
-    Assert.assertEquals(doubleData.getNumDVs(), doubleDataFromDB.getNumDVs());
-    Assert.assertEquals(doubleData.getBitVectors(), doubleDataFromDB.getBitVectors());
-  }
-
-  @Test
-  public void stringPartitionStatistics() throws Exception {
-    createMockTableAndPartition(STRING_TYPE, STRING_VAL);
-    // Add partition stats for: STRING_COL and partition: {PART_KEY, STRING_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, STRING_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = stringColStatsObjs.get(0);
-    StringColumnStatsData stringData = obj.getStatsData().getStringStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(STRING_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, dataFromDB.getSetField());
-    // Compare StringColumnStatsData
-    StringColumnStatsData stringDataFromDB = dataFromDB.getStringStats();
-    Assert.assertEquals(stringData.getMaxColLen(), stringDataFromDB.getMaxColLen());
-    Assert.assertEquals(stringData.getAvgColLen(), stringDataFromDB.getAvgColLen(), 0.01);
-    Assert.assertEquals(stringData.getNumNulls(), stringDataFromDB.getNumNulls());
-    Assert.assertEquals(stringData.getNumDVs(), stringDataFromDB.getNumDVs());
-    Assert.assertEquals(stringData.getBitVectors(), stringDataFromDB.getBitVectors());
-  }
-
-  @Test
-  public void decimalPartitionStatistics() throws Exception {
-    createMockTableAndPartition(DECIMAL_TYPE, DECIMAL_VAL);
-    // Add partition stats for: DECIMAL_COL and partition: {PART_KEY, DECIMAL_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, DECIMAL_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = decimalColStatsObjs.get(0);
-    DecimalColumnStatsData decimalData = obj.getStatsData().getDecimalStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(DECIMAL_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, dataFromDB.getSetField());
-    // Compare DecimalColumnStatsData
-    DecimalColumnStatsData decimalDataFromDB = dataFromDB.getDecimalStats();
-    Assert.assertEquals(decimalData.getHighValue(), decimalDataFromDB.getHighValue());
-    Assert.assertEquals(decimalData.getLowValue(), decimalDataFromDB.getLowValue());
-    Assert.assertEquals(decimalData.getNumNulls(), decimalDataFromDB.getNumNulls());
-    Assert.assertEquals(decimalData.getNumDVs(), decimalDataFromDB.getNumDVs());
-    Assert.assertEquals(decimalData.getBitVectors(), decimalDataFromDB.getBitVectors());
-  }
-
-  private Table createMockTable(String name, String type) throws Exception {
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema(name, type, ""));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    Map<String, String> params = new HashMap<String, String>();
-    params.put("key", "value");
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
-        serde, new ArrayList<String>(), new ArrayList<Order>(), params);
-    int currentTime = (int)(System.currentTimeMillis() / 1000);
-    Table table = new Table(TBL, DB, "me", currentTime, currentTime, 0, sd, cols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-    return table;
-  }
-
-  private Table createMockTableAndPartition(String partType, String partVal) throws Exception {
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", partType, ""));
-    List<String> vals = new ArrayList<String>();
-    vals.add(partVal);
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    Map<String, String> params = new HashMap<String, String>();
-    params.put("key", "value");
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
-        serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
-    int currentTime = (int)(System.currentTimeMillis() / 1000);
-    Table table = new Table(TBL, DB, "me", currentTime, currentTime, 0, sd, cols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-    Partition part = new Partition(vals, DB, TBL, currentTime, currentTime, sd,
-        emptyParameters);
-    store.addPartition(part);
-    return table;
-  }
-  /**
-   * Returns a dummy table level ColumnStatisticsDesc with default values
-   */
-  private ColumnStatisticsDesc getMockTblColStatsDesc() {
-    ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
-    desc.setLastAnalyzed(DEFAULT_TIME);
-    desc.setDbName(DB);
-    desc.setTableName(TBL);
-    desc.setIsTblLevel(true);
-    return desc;
-  }
-
-  /**
-   * Returns a dummy partition level ColumnStatisticsDesc
-   */
-  private ColumnStatisticsDesc getMockPartColStatsDesc(String partKey, String partVal) {
-    ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
-    desc.setLastAnalyzed(DEFAULT_TIME);
-    desc.setDbName(DB);
-    desc.setTableName(TBL);
-    // part1=val1
-    desc.setPartName(partKey + PART_KV_SEPARATOR + partVal);
-    desc.setIsTblLevel(false);
-    return desc;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
deleted file mode 100644
index 4ccb7dd..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
+++ /dev/null
@@ -1,365 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class TestHBaseStoreCached {
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreCached.class.getName());
-  static Map<String, String> emptyParameters = new HashMap<String, String>();
-
-  @Rule public ExpectedException thrown = ExpectedException.none();
-  @Mock HTableInterface htable;
-  SortedMap<String, Cell> rows = new TreeMap<String, Cell>();
-  HBaseStore store;
-
-  @Before
-  public void init() throws IOException {
-    MockitoAnnotations.initMocks(this);
-    HiveConf conf = new HiveConf();
-    store = MockUtils.init(conf, htable, rows);
-  }
-
-  @Test
-  public void createTable() throws Exception {
-    String tableName = "mytable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    Table t = store.getTable("default", tableName);
-    Assert.assertEquals(1, t.getSd().getColsSize());
-    Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
-    Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp", t.getSd().getLocation());
-    Assert.assertEquals("input", t.getSd().getInputFormat());
-    Assert.assertEquals("output", t.getSd().getOutputFormat());
-    Assert.assertEquals("me", t.getOwner());
-    Assert.assertEquals("default", t.getDbName());
-    Assert.assertEquals(tableName, t.getTableName());
-  }
-
-  @Test
-  public void alterTable() throws Exception {
-    String tableName = "alttable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    startTime += 10;
-    table.setLastAccessTime(startTime);
-    store.alterTable("default", tableName, table);
-
-    Table t = store.getTable("default", tableName);
-    Assert.assertEquals(1, t.getSd().getColsSize());
-    Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
-    Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp", t.getSd().getLocation());
-    Assert.assertEquals("input", t.getSd().getInputFormat());
-    Assert.assertEquals("output", t.getSd().getOutputFormat());
-    Assert.assertEquals("me", t.getOwner());
-    Assert.assertEquals("default", t.getDbName());
-    Assert.assertEquals(tableName, t.getTableName());
-    Assert.assertEquals(startTime, t.getLastAccessTime());
-  }
-
-  @Test
-  public void dropTable() throws Exception {
-    String tableName = "dtable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    Table t = store.getTable("default", tableName);
-    Assert.assertNotNull(t);
-
-    store.dropTable("default", tableName);
-    Assert.assertNull(store.getTable("default", tableName));
-  }
-
-  @Test
-  public void createPartition() throws Exception {
-    String dbName = "default";
-    String tableName = "myparttable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> vals = Arrays.asList("fred");
-    StorageDescriptor psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/pc=fred");
-    Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-        emptyParameters);
-    store.addPartition(part);
-
-    Partition p = store.getPartition(dbName, tableName, vals);
-    Assert.assertEquals(1, p.getSd().getColsSize());
-    Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
-    Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
-    Assert.assertEquals("input", p.getSd().getInputFormat());
-    Assert.assertEquals("output", p.getSd().getOutputFormat());
-    Assert.assertEquals(dbName, p.getDbName());
-    Assert.assertEquals(tableName, p.getTableName());
-    Assert.assertEquals(1, p.getValuesSize());
-    Assert.assertEquals("fred", p.getValues().get(0));
-  }
-
-  @Test
-  public void getPartitions() throws Exception {
-    String dbName = "default";
-    String tableName = "manyParts";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
-    for (String val : partVals) {
-      List<String> vals = new ArrayList<String>();
-      vals.add(val);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/pc=" + val);
-      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-          emptyParameters);
-      store.addPartition(part);
-
-      Partition p = store.getPartition(dbName, tableName, vals);
-      Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
-    }
-
-    List<Partition> parts = store.getPartitions(dbName, tableName, -1);
-    Assert.assertEquals(5, parts.size());
-    String[] pv = new String[5];
-    for (int i = 0; i < 5; i++) pv[i] = parts.get(i).getValues().get(0);
-    Arrays.sort(pv);
-    Assert.assertArrayEquals(pv, partVals.toArray(new String[5]));
-  }
-
-  @Test
-  public void listGetDropPartitionNames() throws Exception {
-    String dbName = "default";
-    String tableName = "listParts";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    partCols.add(new FieldSchema("region", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    String[][] partVals = new String[][]{{"today", "north america"}, {"tomorrow", "europe"}};
-    for (String[] pv : partVals) {
-      List<String> vals = new ArrayList<String>();
-      for (String v : pv) vals.add(v);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/pc=" + pv[0] + "/region=" + pv[1]);
-      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-          emptyParameters);
-      store.addPartition(part);
-    }
-
-    List<String> names = store.listPartitionNames(dbName, tableName, (short) -1);
-    Assert.assertEquals(2, names.size());
-    String[] resultNames = names.toArray(new String[names.size()]);
-    Arrays.sort(resultNames);
-    Assert.assertArrayEquals(resultNames, new String[]{"pc=today/region=north america",
-        "pc=tomorrow/region=europe"});
-
-    List<Partition> parts = store.getPartitionsByNames(dbName, tableName, names);
-    Assert.assertArrayEquals(partVals[0], parts.get(0).getValues().toArray(new String[2]));
-    Assert.assertArrayEquals(partVals[1], parts.get(1).getValues().toArray(new String[2]));
-
-    store.dropPartitions(dbName, tableName, names);
-    List<Partition> afterDropParts = store.getPartitions(dbName, tableName, -1);
-    Assert.assertEquals(0, afterDropParts.size());
-  }
-
-
-  @Test
-  public void dropPartition() throws Exception {
-    String dbName = "default";
-    String tableName = "myparttable2";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> vals = Arrays.asList("fred");
-    StorageDescriptor psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/pc=fred");
-    Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-        emptyParameters);
-    store.addPartition(part);
-
-    Assert.assertNotNull(store.getPartition(dbName, tableName, vals));
-    store.dropPartition(dbName, tableName, vals);
-    thrown.expect(NoSuchObjectException.class);
-    store.getPartition(dbName, tableName, vals);
-  }
-
-  // Due to the way our mock stuff works, we can only insert one column at a time, so we'll test
-  // each stat type separately.  We'll test them together in hte integration tests.
-  @Test
-  public void booleanTableStatistics() throws Exception {
-    long now = System.currentTimeMillis();
-    String dbname = "default";
-    String tableName = "statstable";
-    String boolcol = "boolcol";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema(boolcol, "boolean", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table(tableName, dbname, "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    long trues = 37;
-    long falses = 12;
-    long booleanNulls = 2;
-
-    ColumnStatistics stats = new ColumnStatistics();
-    ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
-    desc.setLastAnalyzed(now);
-    desc.setDbName(dbname);
-    desc.setTableName(tableName);
-    desc.setIsTblLevel(true);
-    stats.setStatsDesc(desc);
-
-    ColumnStatisticsObj obj = new ColumnStatisticsObj();
-    obj.setColName(boolcol);
-    obj.setColType("boolean");
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    BooleanColumnStatsData boolData = new BooleanColumnStatsData();
-    boolData.setNumTrues(trues);
-    boolData.setNumFalses(falses);
-    boolData.setNumNulls(booleanNulls);
-    data.setBooleanStats(boolData);
-    obj.setStatsData(data);
-    stats.addToStatsObj(obj);
-
-    store.updateTableColumnStatistics(stats);
-
-    stats = store.getTableColumnStatistics(dbname, tableName, Arrays.asList(boolcol));
-    Assert.assertEquals(now, stats.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(dbname, stats.getStatsDesc().getDbName());
-    Assert.assertEquals(tableName, stats.getStatsDesc().getTableName());
-    Assert.assertTrue(stats.getStatsDesc().isIsTblLevel());
-
-    Assert.assertEquals(1, stats.getStatsObjSize());
-    ColumnStatisticsData colData = obj.getStatsData();
-    Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS, colData.getSetField());
-    boolData = colData.getBooleanStats();
-    Assert.assertEquals(trues, boolData.getNumTrues());
-    Assert.assertEquals(falses, boolData.getNumFalses());
-    Assert.assertEquals(booleanNulls, boolData.getNumNulls());
-  }
-
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java
deleted file mode 100644
index bea0b34..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- *
- */
-public class TestSharedStorageDescriptor {
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStore.class.getName());
-
-
-  @Test
-  public void changeOnSerde() {
-    StorageDescriptor sd = new StorageDescriptor();
-    SerDeInfo serde = new SerDeInfo();
-    serde.setName("serde");
-    sd.setSerdeInfo(serde);
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setShared(sd);
-    ssd.getSerdeInfo().setName("different");
-    Assert.assertFalse(sd.getSerdeInfo() == ssd.getSerdeInfo());
-    Assert.assertEquals("serde", serde.getName());
-    Assert.assertEquals("different", ssd.getSerdeInfo().getName());
-    Assert.assertEquals("serde", sd.getSerdeInfo().getName());
-  }
-
-  @Test
-  public void changeOnSkewed() {
-    StorageDescriptor sd = new StorageDescriptor();
-    SkewedInfo skew = new SkewedInfo();
-    sd.setSkewedInfo(skew);
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setShared(sd);
-    ssd.setSkewedInfo(new SkewedInfo());
-    Assert.assertFalse(sd.getSkewedInfo() == ssd.getSkewedInfo());
-  }
-
-  @Test
-  public void changeOnUnset() {
-    StorageDescriptor sd = new StorageDescriptor();
-    SkewedInfo skew = new SkewedInfo();
-    sd.setSkewedInfo(skew);
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setShared(sd);
-    ssd.unsetSkewedInfo();
-    Assert.assertFalse(sd.getSkewedInfo() == ssd.getSkewedInfo());
-  }
-
-  @Test
-  public void changeOrder() {
-    StorageDescriptor sd = new StorageDescriptor();
-    sd.addToSortCols(new Order("fred", 1));
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setShared(sd);
-    ssd.getSortCols().get(0).setOrder(2);
-    Assert.assertFalse(sd.getSortCols() == ssd.getSortCols());
-    Assert.assertEquals(2, ssd.getSortCols().get(0).getOrder());
-    Assert.assertEquals(1, sd.getSortCols().get(0).getOrder());
-  }
-
-  @Test
-  public void unsetOrder() {
-    StorageDescriptor sd = new StorageDescriptor();
-    sd.addToSortCols(new Order("fred", 1));
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setShared(sd);
-    ssd.unsetSortCols();
-    Assert.assertFalse(sd.getSortCols() == ssd.getSortCols());
-    Assert.assertEquals(0, ssd.getSortColsSize());
-    Assert.assertEquals(1, sd.getSortColsSize());
-  }
-
-  @Test
-  public void changeBucketList() {
-    StorageDescriptor sd = new StorageDescriptor();
-    sd.addToBucketCols(new String("fred"));
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setShared(sd);
-    List<String> list = ssd.getBucketCols();
-    list.add(new String("bob"));
-    Assert.assertFalse(sd.getBucketCols() == ssd.getBucketCols());
-    Assert.assertEquals(2, ssd.getBucketColsSize());
-    Assert.assertEquals(1, sd.getBucketColsSize());
-  }
-
-  @Test
-  public void addToColList() {
-    StorageDescriptor sd = new StorageDescriptor();
-    sd.addToCols(new FieldSchema("fred", "string", ""));
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setShared(sd);
-    ssd.addToCols(new FieldSchema("joe", "int", ""));
-    Assert.assertFalse(sd.getCols() == ssd.getCols());
-    Assert.assertEquals(2, ssd.getColsSize());
-    Assert.assertEquals(1, sd.getColsSize());
-  }
-
-  @Test
-  public void colIterator() {
-    StorageDescriptor sd = new StorageDescriptor();
-    sd.addToCols(new FieldSchema("fred", "string", ""));
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setShared(sd);
-    Iterator<FieldSchema> iter = ssd.getColsIterator();
-    Assert.assertTrue(iter.hasNext());
-    Assert.assertEquals("fred", iter.next().getName());
-    Assert.assertFalse(sd.getCols() == ssd.getCols());
-  }
-
-  @Test
-  public void setReadOnly() {
-    StorageDescriptor sd = new StorageDescriptor();
-    sd.addToCols(new FieldSchema("fred", "string", ""));
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setShared(sd);
-    ssd.setReadOnly();
-    List<FieldSchema> cols = ssd.getCols();
-    Assert.assertEquals(1, cols.size());
-    Assert.assertTrue(sd.getCols() == ssd.getCols());
-  }
-
-}


[08/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
deleted file mode 100644
index d01f814..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ /dev/null
@@ -1,2969 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.CacheLoader;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.FileMetadataHandler;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.PartFilterExprUtil;
-import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.HiveObjectType;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.InvalidInputException;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionEventType;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TableMeta;
-import org.apache.hadoop.hive.metastore.api.Type;
-import org.apache.hadoop.hive.metastore.api.UnknownDBException;
-import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
-import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
-import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hive.common.util.HiveStringUtils;
-import org.apache.thrift.TException;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-/**
- * Implementation of RawStore that stores data in HBase
- */
-public class HBaseStore implements RawStore {
-  static final private Logger LOG = LoggerFactory.getLogger(HBaseStore.class.getName());
-
-  // Do not access this directly, call getHBase to make sure it is initialized.
-  private HBaseReadWrite hbase = null;
-  private Configuration conf;
-  private int txnNestLevel = 0;
-  private PartitionExpressionProxy expressionProxy = null;
-  private Map<FileMetadataExprType, FileMetadataHandler> fmHandlers;
-
-  public HBaseStore() {
-  }
-
-  @Override
-  public void shutdown() {
-    try {
-      if (txnNestLevel != 0) rollbackTransaction();
-      getHBase().close();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
-  public boolean openTransaction() {
-    if (txnNestLevel++ <= 0) {
-      LOG.debug("Opening HBase transaction");
-      getHBase().begin();
-      txnNestLevel = 1;
-    }
-    return true;
-  }
-
-  @Override
-  public boolean commitTransaction() {
-    if (--txnNestLevel == 0) {
-      LOG.debug("Committing HBase transaction");
-      getHBase().commit();
-    }
-    return true;
-  }
-
-  @Override
-  public boolean isActiveTransaction() {
-    return txnNestLevel != 0;
-  }
-
-  @Override
-  public void rollbackTransaction() {
-    txnNestLevel = 0;
-    LOG.debug("Rolling back HBase transaction");
-    getHBase().rollback();
-  }
-
-  @Override
-  public void createDatabase(Database db) throws InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Database dbCopy = db.deepCopy();
-      dbCopy.setName(HiveStringUtils.normalizeIdentifier(dbCopy.getName()));
-      // HiveMetaStore already checks for existence of the database, don't recheck
-      getHBase().putDb(dbCopy);
-      commit = true;
-    } catch (IOException e) {
-      LOG.error("Unable to create database ", e);
-      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-
-  }
-
-  @Override
-  public Database getDatabase(String name) throws NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Database db = getHBase().getDb(HiveStringUtils.normalizeIdentifier(name));
-      if (db == null) {
-        throw new NoSuchObjectException("Unable to find db " + name);
-      }
-      commit = true;
-      return db;
-    } catch (IOException e) {
-      LOG.error("Unable to get db", e);
-      throw new NoSuchObjectException("Error reading db " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      getHBase().deleteDb(HiveStringUtils.normalizeIdentifier(dbname));
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to delete db" + e);
-      throw new MetaException("Unable to drop database " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean alterDatabase(String dbname, Database db) throws NoSuchObjectException,
-      MetaException {
-    // ObjectStore fetches the old db before updating it, but I can't see the possible value of
-    // that since the caller will have needed to call getDatabase to have the db object.
-    boolean commit = false;
-    openTransaction();
-    try {
-      Database dbCopy = db.deepCopy();
-      dbCopy.setName(HiveStringUtils.normalizeIdentifier(dbCopy.getName()));
-      getHBase().putDb(dbCopy);
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to alter database ", e);
-      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<String> getDatabases(String pattern) throws MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Database> dbs = getHBase().scanDatabases(
-          pattern==null?null:HiveStringUtils.normalizeIdentifier(likeToRegex(pattern)));
-      List<String> dbNames = new ArrayList<String>(dbs.size());
-      for (Database db : dbs) dbNames.add(db.getName());
-      commit = true;
-      return dbNames;
-    } catch (IOException e) {
-      LOG.error("Unable to get databases ", e);
-      throw new MetaException("Unable to get databases, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<String> getAllDatabases() throws MetaException {
-    return getDatabases(null);
-  }
-
-  @Override
-  public int getDatabaseCount() throws MetaException {
-    try {
-      return getHBase().getDatabaseCount();
-    } catch (IOException e) {
-      LOG.error("Unable to get database count", e);
-      throw new MetaException("Error scanning databases");
-    }
-  }
-
-  @Override
-  public boolean createType(Type type) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public Type getType(String typeName) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public boolean dropType(String typeName) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public void createTable(Table tbl) throws InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    // HiveMetaStore above us checks if the table already exists, so we can blindly store it here.
-    try {
-      Table tblCopy = tbl.deepCopy();
-      tblCopy.setDbName(HiveStringUtils.normalizeIdentifier(tblCopy.getDbName()));
-      tblCopy.setTableName(HiveStringUtils.normalizeIdentifier(tblCopy.getTableName()));
-      normalizeColumnNames(tblCopy);
-      getHBase().putTable(tblCopy);
-      commit = true;
-    } catch (IOException e) {
-      LOG.error("Unable to create table ", e);
-      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  private void normalizeColumnNames(Table tbl) {
-    if (tbl.getSd().getCols() != null) {
-      tbl.getSd().setCols(normalizeFieldSchemaList(tbl.getSd().getCols()));
-    }
-    if (tbl.getPartitionKeys() != null) {
-      tbl.setPartitionKeys(normalizeFieldSchemaList(tbl.getPartitionKeys()));
-    }
-  }
-
-  private List<FieldSchema> normalizeFieldSchemaList(List<FieldSchema> fieldschemas) {
-    List<FieldSchema> ret = new ArrayList<>();
-    for (FieldSchema fieldSchema : fieldschemas) {
-      ret.add(new FieldSchema(fieldSchema.getName().toLowerCase(), fieldSchema.getType(),
-          fieldSchema.getComment()));
-    }
-    return ret;
-  }
-
-  @Override
-  public boolean dropTable(String dbName, String tableName) throws MetaException,
-      NoSuchObjectException, InvalidObjectException, InvalidInputException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      getHBase().deleteTable(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(tableName));
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to delete db" + e);
-      throw new MetaException("Unable to drop table " + tableNameForErrorMsg(dbName, tableName));
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public Table getTable(String dbName, String tableName) throws MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Table table = getHBase().getTable(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(tableName));
-      if (table == null) {
-        LOG.debug("Unable to find table " + tableNameForErrorMsg(dbName, tableName));
-      }
-      commit = true;
-      return table;
-    } catch (IOException e) {
-      LOG.error("Unable to get table", e);
-      throw new MetaException("Error reading table " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean addPartition(Partition part) throws InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Partition partCopy = part.deepCopy();
-      partCopy.setDbName(HiveStringUtils.normalizeIdentifier(part.getDbName()));
-      partCopy.setTableName(HiveStringUtils.normalizeIdentifier(part.getTableName()));
-      getHBase().putPartition(partCopy);
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to add partition", e);
-      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean addPartitions(String dbName, String tblName, List<Partition> parts)
-      throws InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Partition> partsCopy = new ArrayList<Partition>();
-      for (int i=0;i<parts.size();i++) {
-        Partition partCopy = parts.get(i).deepCopy();
-        partCopy.setDbName(HiveStringUtils.normalizeIdentifier(partCopy.getDbName()));
-        partCopy.setTableName(HiveStringUtils.normalizeIdentifier(partCopy.getTableName()));
-        partsCopy.add(i, partCopy);
-      }
-      getHBase().putPartitions(partsCopy);
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to add partitions", e);
-      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean addPartitions(String dbName, String tblName, PartitionSpecProxy partitionSpec,
-                               boolean ifNotExists) throws InvalidObjectException, MetaException {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public Partition getPartition(String dbName, String tableName, List<String> part_vals) throws
-      MetaException, NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Partition part = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(tableName), part_vals);
-      if (part == null) {
-        throw new NoSuchObjectException("Unable to find partition " +
-            partNameForErrorMsg(dbName, tableName, part_vals));
-      }
-      commit = true;
-      return part;
-    } catch (IOException e) {
-      LOG.error("Unable to get partition", e);
-      throw new MetaException("Error reading partition " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean doesPartitionExist(String dbName, String tableName, List<String> part_vals) throws
-      MetaException, NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      boolean exists = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(tableName), part_vals) != null;
-      commit = true;
-      return exists;
-    } catch (IOException e) {
-      LOG.error("Unable to get partition", e);
-      throw new MetaException("Error reading partition " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean dropPartition(String dbName, String tableName, List<String> part_vals) throws
-      MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      dbName = HiveStringUtils.normalizeIdentifier(dbName);
-      tableName = HiveStringUtils.normalizeIdentifier(tableName);
-      getHBase().deletePartition(dbName, tableName, HBaseUtils.getPartitionKeyTypes(
-          getTable(dbName, tableName).getPartitionKeys()), part_vals);
-      // Drop any cached stats that reference this partitions
-      getHBase().getStatsCache().invalidate(dbName, tableName,
-          buildExternalPartName(dbName, tableName, part_vals));
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to delete db" + e);
-      throw new MetaException("Unable to drop partition " + partNameForErrorMsg(dbName, tableName,
-          part_vals));
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<Partition> getPartitions(String dbName, String tableName, int max) throws
-      MetaException, NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Partition> parts = getHBase().scanPartitionsInTable(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(tableName), max);
-      commit = true;
-      return parts;
-    } catch (IOException e) {
-      LOG.error("Unable to get partitions", e);
-      throw new MetaException("Error scanning partitions");
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public void alterTable(String dbName, String tableName, Table newTable) throws InvalidObjectException,
-      MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Table newTableCopy = newTable.deepCopy();
-      newTableCopy.setDbName(HiveStringUtils.normalizeIdentifier(newTableCopy.getDbName()));
-      List<String> oldPartTypes = getTable(dbName, tableName).getPartitionKeys()==null?
-          null:HBaseUtils.getPartitionKeyTypes(getTable(dbName, tableName).getPartitionKeys());
-      newTableCopy.setTableName(HiveStringUtils.normalizeIdentifier(newTableCopy.getTableName()));
-      getHBase().replaceTable(getHBase().getTable(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(tableName)), newTableCopy);
-      if (newTable.getPartitionKeys() != null && newTable.getPartitionKeys().size() > 0
-          && !tableName.equals(newTable.getTableName())) {
-        // They renamed the table, so we need to change each partition as well, since it changes
-        // the key.
-        try {
-          List<Partition> oldParts = getPartitions(dbName, tableName, -1);
-          List<Partition> newParts = new ArrayList<>(oldParts.size());
-          for (Partition oldPart : oldParts) {
-            Partition newPart = oldPart.deepCopy();
-            newPart.setTableName(newTable.getTableName());
-            newParts.add(newPart);
-          }
-          getHBase().replacePartitions(oldParts, newParts, oldPartTypes);
-        } catch (NoSuchObjectException e) {
-          LOG.debug("No partitions found for old table so not worrying about it");
-        }
-
-      }
-      commit = true;
-    } catch (IOException e) {
-      LOG.error("Unable to alter table " + tableNameForErrorMsg(dbName, tableName), e);
-      throw new MetaException("Unable to alter table " + tableNameForErrorMsg(dbName, tableName));
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<String> getTables(String dbName, String pattern) throws MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<String> tableNames = getTableNamesInTx(dbName, pattern);
-      commit = true;
-      return tableNames;
-    } catch (IOException e) {
-      LOG.error("Unable to get tables ", e);
-      throw new MetaException("Unable to get tables, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<String> getTables(String dbName, String pattern, TableType tableType) throws MetaException {
-    throw new UnsupportedOperationException();
-  }
-
-  private List<String> getTableNamesInTx(String dbName, String pattern) throws IOException {
-    List<Table> tables = getHBase().scanTables(HiveStringUtils.normalizeIdentifier(dbName),
-        pattern==null?null:HiveStringUtils.normalizeIdentifier(likeToRegex(pattern)));
-    List<String> tableNames = new ArrayList<String>(tables.size());
-    for (Table table : tables) tableNames.add(table.getTableName());
-    return tableNames;
-  }
-
-  @Override
-  public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes)
-      throws MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<TableMeta> metas = new ArrayList<>();
-      for (String dbName : getDatabases(dbNames)) {
-        for (Table table : getTableObjectsByName(dbName, getTableNamesInTx(dbName, tableNames))) {
-          if (tableTypes == null || tableTypes.contains(table.getTableType())) {
-            TableMeta metaData = new TableMeta(
-              table.getDbName(), table.getTableName(), table.getTableType());
-            metaData.setComments(table.getParameters().get("comment"));
-            metas.add(metaData);
-          }
-        }
-      }
-      commit = true;
-      return metas;
-    } catch (Exception e) {
-      LOG.error("Unable to get tables ", e);
-      throw new MetaException("Unable to get tables, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<Table> getTableObjectsByName(String dbname, List<String> tableNames) throws
-      MetaException, UnknownDBException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<String> normalizedTableNames = new ArrayList<String>(tableNames.size());
-      for (String tableName : tableNames) {
-        normalizedTableNames.add(HiveStringUtils.normalizeIdentifier(tableName));
-      }
-      List<Table> tables = getHBase().getTables(HiveStringUtils.normalizeIdentifier(dbname),
-          normalizedTableNames);
-      commit = true;
-      return tables;
-    } catch (IOException e) {
-      LOG.error("Unable to get tables ", e);
-      throw new MetaException("Unable to get tables, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<String> getAllTables(String dbName) throws MetaException {
-    return getTables(dbName, null);
-  }
-
-  @Override
-  public int getTableCount() throws MetaException {
-    try {
-      return getHBase().getTableCount();
-    } catch (IOException e) {
-      LOG.error("Unable to get table count", e);
-      throw new MetaException("Error scanning tables");
-    }
-  }
-
-  @Override
-  public List<String> listTableNamesByFilter(String dbName, String filter, short max_tables) throws
-      MetaException, UnknownDBException {
-    // TODO needs to wait until we support pushing filters into HBase.
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public List<String> listPartitionNames(String db_name, String tbl_name, short max_parts) throws
-      MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Partition> parts = getHBase().scanPartitionsInTable(HiveStringUtils.normalizeIdentifier(db_name),
-          HiveStringUtils.normalizeIdentifier(tbl_name), max_parts);
-      if (parts == null) return null;
-      List<String> names = new ArrayList<String>(parts.size());
-      Table table = getHBase().getTable(HiveStringUtils.normalizeIdentifier(db_name),
-          HiveStringUtils.normalizeIdentifier(tbl_name));
-      for (Partition p : parts) {
-        names.add(buildExternalPartName(table, p));
-      }
-      commit = true;
-      return names;
-    } catch (IOException e) {
-      LOG.error("Unable to get partitions", e);
-      throw new MetaException("Error scanning partitions");
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<String> listPartitionNamesByFilter(String db_name, String tbl_name, String filter,
-                                                 short max_parts) throws MetaException {
-    // TODO needs to wait until we support pushing filters into HBase.
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public void alterPartition(String db_name, String tbl_name, List<String> part_vals,
-                             Partition new_part) throws InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Partition new_partCopy = new_part.deepCopy();
-      new_partCopy.setDbName(HiveStringUtils.normalizeIdentifier(new_partCopy.getDbName()));
-      new_partCopy.setTableName(HiveStringUtils.normalizeIdentifier(new_partCopy.getTableName()));
-      Partition oldPart = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(db_name),
-          HiveStringUtils.normalizeIdentifier(tbl_name), part_vals);
-      getHBase().replacePartition(oldPart, new_partCopy, HBaseUtils.getPartitionKeyTypes(
-          getTable(db_name, tbl_name).getPartitionKeys()));
-      // Drop any cached stats that reference this partitions
-      getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(db_name),
-          HiveStringUtils.normalizeIdentifier(tbl_name),
-          buildExternalPartName(db_name, tbl_name, part_vals));
-      commit = true;
-    } catch (IOException e) {
-      LOG.error("Unable to add partition", e);
-      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public void alterPartitions(String db_name, String tbl_name, List<List<String>> part_vals_list,
-                              List<Partition> new_parts) throws InvalidObjectException,
-      MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Partition> new_partsCopy = new ArrayList<Partition>();
-      for (int i=0;i<new_parts.size();i++) {
-        Partition newPartCopy = new_parts.get(i).deepCopy();
-        newPartCopy.setDbName(HiveStringUtils.normalizeIdentifier(newPartCopy.getDbName()));
-        newPartCopy.setTableName(HiveStringUtils.normalizeIdentifier(newPartCopy.getTableName()));
-        new_partsCopy.add(i, newPartCopy);
-      }
-      List<Partition> oldParts = getHBase().getPartitions(HiveStringUtils.normalizeIdentifier(db_name),
-          HiveStringUtils.normalizeIdentifier(tbl_name),
-          HBaseUtils.getPartitionKeyTypes(getTable(HiveStringUtils.normalizeIdentifier(db_name),
-          HiveStringUtils.normalizeIdentifier(tbl_name)).getPartitionKeys()), part_vals_list);
-      getHBase().replacePartitions(oldParts, new_partsCopy, HBaseUtils.getPartitionKeyTypes(
-          getTable(db_name, tbl_name).getPartitionKeys()));
-      for (List<String> part_vals : part_vals_list) {
-        getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(db_name),
-            HiveStringUtils.normalizeIdentifier(tbl_name),
-            buildExternalPartName(db_name, tbl_name, part_vals));
-      }
-      commit = true;
-    } catch (IOException e) {
-      LOG.error("Unable to add partition", e);
-      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean addIndex(Index index) throws InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      index.setDbName(HiveStringUtils.normalizeIdentifier(index.getDbName()));
-      index.setOrigTableName(HiveStringUtils.normalizeIdentifier(index.getOrigTableName()));
-      index.setIndexName(HiveStringUtils.normalizeIdentifier(index.getIndexName()));
-      index.setIndexTableName(HiveStringUtils.normalizeIdentifier(index.getIndexTableName()));
-      getHBase().putIndex(index);
-      commit = true;
-    } catch (IOException e) {
-      LOG.error("Unable to create index ", e);
-      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-    return commit;
-  }
-
-  @Override
-  public Index getIndex(String dbName, String origTableName, String indexName) throws
-      MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Index index = getHBase().getIndex(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(origTableName),
-          HiveStringUtils.normalizeIdentifier(indexName));
-      if (index == null) {
-        LOG.debug("Unable to find index " + indexNameForErrorMsg(dbName, origTableName, indexName));
-      }
-      commit = true;
-      return index;
-    } catch (IOException e) {
-      LOG.error("Unable to get index", e);
-      throw new MetaException("Error reading index " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean dropIndex(String dbName, String origTableName, String indexName) throws
-      MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      getHBase().deleteIndex(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(origTableName),
-          HiveStringUtils.normalizeIdentifier(indexName));
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to delete index" + e);
-      throw new MetaException("Unable to drop index "
-          + indexNameForErrorMsg(dbName, origTableName, indexName));
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<Index> getIndexes(String dbName, String origTableName, int max) throws MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Index> indexes = getHBase().scanIndexes(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(origTableName), max);
-      commit = true;
-      return indexes;
-    } catch (IOException e) {
-      LOG.error("Unable to get indexes", e);
-      throw new MetaException("Error scanning indexxes");
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<String> listIndexNames(String dbName, String origTableName, short max) throws
-      MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Index> indexes = getHBase().scanIndexes(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(origTableName), max);
-      if (indexes == null) return null;
-      List<String> names = new ArrayList<String>(indexes.size());
-      for (Index index : indexes) {
-        names.add(index.getIndexName());
-      }
-      commit = true;
-      return names;
-    } catch (IOException e) {
-      LOG.error("Unable to get indexes", e);
-      throw new MetaException("Error scanning indexes");
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public void alterIndex(String dbname, String baseTblName, String name, Index newIndex) throws
-      InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Index newIndexCopy = newIndex.deepCopy();
-      newIndexCopy.setDbName(HiveStringUtils.normalizeIdentifier(newIndexCopy.getDbName()));
-      newIndexCopy.setOrigTableName(
-          HiveStringUtils.normalizeIdentifier(newIndexCopy.getOrigTableName()));
-      newIndexCopy.setIndexName(HiveStringUtils.normalizeIdentifier(newIndexCopy.getIndexName()));
-      getHBase().replaceIndex(getHBase().getIndex(HiveStringUtils.normalizeIdentifier(dbname),
-          HiveStringUtils.normalizeIdentifier(baseTblName),
-          HiveStringUtils.normalizeIdentifier(name)), newIndexCopy);
-      commit = true;
-    } catch (IOException e) {
-      LOG.error("Unable to alter index " + indexNameForErrorMsg(dbname, baseTblName, name), e);
-      throw new MetaException("Unable to alter index "
-          + indexNameForErrorMsg(dbname, baseTblName, name));
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<Partition> getPartitionsByFilter(String dbName, String tblName, String filter,
-                                               short maxParts) throws MetaException,
-      NoSuchObjectException {
-    final ExpressionTree exprTree = (filter != null && !filter.isEmpty()) ? PartFilterExprUtil
-        .getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE;
-    List<Partition> result = new ArrayList<Partition>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      getPartitionsByExprInternal(HiveStringUtils.normalizeIdentifier(dbName),
-          HiveStringUtils.normalizeIdentifier(tblName), exprTree, maxParts, result);
-      return result;
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr,
-                                     String defaultPartitionName, short maxParts,
-                                     List<Partition> result) throws TException {
-    final ExpressionTree exprTree = PartFilterExprUtil.makeExpressionTree(expressionProxy, expr);
-    dbName = HiveStringUtils.normalizeIdentifier(dbName);
-    tblName = HiveStringUtils.normalizeIdentifier(tblName);
-    Table table = getTable(dbName, tblName);
-    boolean commit = false;
-    openTransaction();
-    try {
-      if (exprTree == null) {
-        List<String> partNames = new LinkedList<String>();
-        boolean hasUnknownPartitions = getPartitionNamesPrunedByExprNoTxn(
-            table, expr, defaultPartitionName, maxParts, partNames);
-        result.addAll(getPartitionsByNames(dbName, tblName, partNames));
-        return hasUnknownPartitions;
-      } else {
-        return getPartitionsByExprInternal(dbName, tblName, exprTree, maxParts, result);
-      }
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public int getNumPartitionsByFilter(String dbName, String tblName, String filter)
-    throws MetaException, NoSuchObjectException {
-    final ExpressionTree exprTree = (filter != null && !filter.isEmpty()) ? PartFilterExprUtil
-      .getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE;
-    List<Partition> result = new ArrayList<Partition>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      return getPartitionsByFilter(dbName, tblName, filter, Short.MAX_VALUE).size();
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public int getNumPartitionsByExpr(String dbName, String tblName, byte[] expr)
-      throws MetaException, NoSuchObjectException {
-    final ExpressionTree exprTree = PartFilterExprUtil.makeExpressionTree(expressionProxy, expr);
-    List<Partition> result = new ArrayList<Partition>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      getPartitionsByExprInternal(dbName, tblName, exprTree, Short.MAX_VALUE, result);
-      return result.size();
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  /**
-   * Gets the partition names from a table, pruned using an expression.
-   * @param table Table.
-   * @param expr Expression.
-   * @param defaultPartName Default partition name from job config, if any.
-   * @param maxParts Maximum number of partition names to return.
-   * @param result The resulting names.
-   * @return Whether the result contains any unknown partitions.
-   * @throws NoSuchObjectException
-   */
-  private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr,
-      String defaultPartName, short maxParts, List<String> result) throws MetaException, NoSuchObjectException {
-    List<Partition> parts = getPartitions(
-        table.getDbName(), table.getTableName(), maxParts);
-    for (Partition part : parts) {
-      result.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues()));
-    }
-    List<String> columnNames = new ArrayList<String>();
-    List<PrimitiveTypeInfo> typeInfos = new ArrayList<PrimitiveTypeInfo>();
-    for (FieldSchema fs : table.getPartitionKeys()) {
-      columnNames.add(fs.getName());
-      typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
-    }
-    if (defaultPartName == null || defaultPartName.isEmpty()) {
-      defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME);
-    }
-    return expressionProxy.filterPartitionsByExpr(
-        columnNames, typeInfos, expr, defaultPartName, result);
-  }
-
-  private boolean getPartitionsByExprInternal(String dbName, String tblName,
-      ExpressionTree exprTree, short maxParts, List<Partition> result) throws MetaException,
-      NoSuchObjectException {
-
-    dbName = HiveStringUtils.normalizeIdentifier(dbName);
-    tblName = HiveStringUtils.normalizeIdentifier(tblName);
-    Table table = getTable(dbName, tblName);
-    if (table == null) {
-      throw new NoSuchObjectException("Unable to find table " + dbName + "." + tblName);
-    }
-    // general hbase filter plan from expression tree
-    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, table.getPartitionKeys());
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Hbase Filter Plan generated : " + planRes.plan);
-    }
-
-    // results from scans need to be merged as there can be overlapping results between
-    // the scans. Use a map of list of partition values to partition for this.
-    Map<List<String>, Partition> mergedParts = new HashMap<List<String>, Partition>();
-    for (ScanPlan splan : planRes.plan.getPlans()) {
-      try {
-        List<Partition> parts = getHBase().scanPartitions(dbName, tblName,
-            splan.getStartRowSuffix(dbName, tblName, table.getPartitionKeys()),
-            splan.getEndRowSuffix(dbName, tblName, table.getPartitionKeys()),
-            splan.getFilter(table.getPartitionKeys()), -1);
-        boolean reachedMax = false;
-        for (Partition part : parts) {
-          mergedParts.put(part.getValues(), part);
-          if (mergedParts.size() == maxParts) {
-            reachedMax = true;
-            break;
-          }
-        }
-        if (reachedMax) {
-          break;
-        }
-      } catch (IOException e) {
-        LOG.error("Unable to get partitions", e);
-        throw new MetaException("Error scanning partitions" + tableNameForErrorMsg(dbName, tblName)
-            + ": " + e);
-      }
-    }
-    for (Entry<List<String>, Partition> mp : mergedParts.entrySet()) {
-      result.add(mp.getValue());
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Matched partitions " + result);
-    }
-
-    // return true if there might be some additional partitions that don't match filter conditions
-    // being returned
-    return !planRes.hasUnsupportedCondition;
-  }
-
-  @Override
-  public List<Partition> getPartitionsByNames(String dbName, String tblName,
-                                              List<String> partNames) throws MetaException,
-      NoSuchObjectException {
-    List<Partition> parts = new ArrayList<Partition>();
-    for (String partName : partNames) {
-      parts.add(getPartition(dbName, tblName, partNameToVals(partName)));
-    }
-    return parts;
-  }
-
-  @Override
-  public Table markPartitionForEvent(String dbName, String tblName, Map<String, String> partVals,
-                                     PartitionEventType evtType) throws MetaException,
-      UnknownTableException, InvalidPartitionException, UnknownPartitionException {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public boolean isPartitionMarkedForEvent(String dbName, String tblName,
-                                           Map<String, String> partName,
-                                           PartitionEventType evtType) throws MetaException,
-      UnknownTableException, InvalidPartitionException, UnknownPartitionException {
-    throw new UnsupportedOperationException();
-  }
-
-  /*
-   * The design for roles.  Roles are a pain because of their hierarchical nature.  When a user
-   * comes in and we need to be able to determine all roles he is a part of, we do not want to
-   * have to walk the hierarchy in the database.  This means we need to flatten the role map for
-   * each user.  But we also have to track how the roles are connected for each user, in case one
-   * role is revoked from another (e.g. if role1 is included in role2 but then revoked
-   * from it and user1 was granted both role2 and role1 we cannot remove user1 from role1
-   * because he was granted that separately).
-   *
-   * We want to optimize for the read case and put the cost on grant and revoke of roles, since
-   * we assume that is the less common case.  So we lay out the roles data as follows:
-   *
-   * There is a ROLES table that records each role, plus what other principals have been granted
-   * into it, along with the info on grantor, etc.
-   *
-   * There is a USER_TO_ROLES table that contains the mapping of each user to every role he is a
-   * part of.
-   *
-   * This makes determining what roles a user participates in very quick, as USER_TO_ROLE is a
-   * simple list for each user.  It makes granting users into roles expensive, and granting roles
-   * into roles very expensive.  Each time a user is granted into a role, we need to walk the
-   * hierarchy in the role table (which means moving through that table multiple times) to
-   * determine every role the user participates in.  Each a role is granted into another role
-   * this hierarchical walk must be done for every principal in the role being granted into.  To
-   * mitigate this pain somewhat whenever doing these mappings we cache the entire ROLES table in
-   * memory since we assume it is not large.
-   *
-   * On a related note, whenever a role is dropped we must walk not only all these role tables
-   * above (equivalent to a role being revoked from another role, since we have to rebuilding
-   * mappings for any users in roles that contained that role and any users directly in that
-   * role), but we also have to remove all the privileges associated with that role directly.
-   * That means a walk of the DBS table and of the TBLS table.
-   */
-
-  @Override
-  public int getPartitionCount() throws MetaException {
-    try {
-      return getHBase().getPartitionCount();
-    } catch (IOException e) {
-      LOG.error("Unable to get partition count", e);
-      throw new MetaException("Error scanning partitions");
-    }
-  }
-
-  @Override
-  public boolean addRole(String roleName, String ownerName) throws InvalidObjectException,
-      MetaException, NoSuchObjectException {
-    int now = (int)(System.currentTimeMillis()/1000);
-    Role role = new Role(roleName, now, ownerName);
-    boolean commit = false;
-    openTransaction();
-    try {
-      if (getHBase().getRole(roleName) != null) {
-        throw new InvalidObjectException("Role " + roleName + " already exists");
-      }
-      getHBase().putRole(role);
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to create role ", e);
-      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean removeRole(String roleName) throws MetaException, NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Set<String> usersInRole = getHBase().findAllUsersInRole(roleName);
-      getHBase().deleteRole(roleName);
-      getHBase().removeRoleGrants(roleName);
-      for (String user : usersInRole) {
-        getHBase().buildRoleMapForUser(user);
-      }
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to delete role" + e);
-      throw new MetaException("Unable to drop role " + roleName);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean grantRole(Role role, String userName, PrincipalType principalType, String grantor,
-                           PrincipalType grantorType, boolean grantOption)
-      throws MetaException, NoSuchObjectException, InvalidObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Set<String> usersToRemap = findUsersToRemapRolesFor(role, userName, principalType);
-      HbaseMetastoreProto.RoleGrantInfo.Builder builder =
-          HbaseMetastoreProto.RoleGrantInfo.newBuilder();
-      if (userName != null) builder.setPrincipalName(userName);
-      if (principalType != null) {
-        builder.setPrincipalType(HBaseUtils.convertPrincipalTypes(principalType));
-      }
-      builder.setAddTime((int)(System.currentTimeMillis() / 1000));
-      if (grantor != null) builder.setGrantor(grantor);
-      if (grantorType != null) {
-        builder.setGrantorType(HBaseUtils.convertPrincipalTypes(grantorType));
-      }
-      builder.setGrantOption(grantOption);
-
-      getHBase().addPrincipalToRole(role.getRoleName(), builder.build());
-      for (String user : usersToRemap) {
-        getHBase().buildRoleMapForUser(user);
-      }
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to grant role", e);
-      throw new MetaException("Unable to grant role " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean revokeRole(Role role, String userName, PrincipalType principalType,
-                            boolean grantOption) throws MetaException, NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    // This can have a couple of different meanings.  If grantOption is true, then this is only
-    // revoking the grant option, the role itself doesn't need to be removed.  If it is false
-    // then we need to remove the userName from the role altogether.
-    try {
-      if (grantOption) {
-        // If this is a grant only change, we don't need to rebuild the user mappings.
-        getHBase().dropPrincipalFromRole(role.getRoleName(), userName, principalType, grantOption);
-      } else {
-        Set<String> usersToRemap = findUsersToRemapRolesFor(role, userName, principalType);
-        getHBase().dropPrincipalFromRole(role.getRoleName(), userName, principalType, grantOption);
-        for (String user : usersToRemap) {
-          getHBase().buildRoleMapForUser(user);
-        }
-      }
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to revoke role " + role.getRoleName() + " from " + userName, e);
-      throw new MetaException("Unable to revoke role " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public PrincipalPrivilegeSet getUserPrivilegeSet(String userName, List<String> groupNames)
-      throws InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      PrincipalPrivilegeSet pps = new PrincipalPrivilegeSet();
-      PrincipalPrivilegeSet global = getHBase().getGlobalPrivs();
-      if (global == null) return null;
-      List<PrivilegeGrantInfo> pgi;
-      if (global.getUserPrivileges() != null) {
-        pgi = global.getUserPrivileges().get(userName);
-        if (pgi != null) {
-          pps.putToUserPrivileges(userName, pgi);
-        }
-      }
-
-      if (global.getRolePrivileges() != null) {
-        List<String> roles = getHBase().getUserRoles(userName);
-        if (roles != null) {
-          for (String role : roles) {
-            pgi = global.getRolePrivileges().get(role);
-            if (pgi != null) {
-              pps.putToRolePrivileges(role, pgi);
-            }
-          }
-        }
-      }
-      commit = true;
-      return pps;
-    } catch (IOException e) {
-      LOG.error("Unable to get db privileges for user", e);
-      throw new MetaException("Unable to get db privileges for user, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public PrincipalPrivilegeSet getDBPrivilegeSet(String dbName, String userName,
-                                                 List<String> groupNames)
-      throws InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      PrincipalPrivilegeSet pps = new PrincipalPrivilegeSet();
-      Database db = getHBase().getDb(dbName);
-      if (db.getPrivileges() != null) {
-        List<PrivilegeGrantInfo> pgi;
-        // Find the user privileges for this db
-        if (db.getPrivileges().getUserPrivileges() != null) {
-          pgi = db.getPrivileges().getUserPrivileges().get(userName);
-          if (pgi != null) {
-            pps.putToUserPrivileges(userName, pgi);
-          }
-        }
-
-        if (db.getPrivileges().getRolePrivileges() != null) {
-          List<String> roles = getHBase().getUserRoles(userName);
-          if (roles != null) {
-            for (String role : roles) {
-              pgi = db.getPrivileges().getRolePrivileges().get(role);
-              if (pgi != null) {
-                pps.putToRolePrivileges(role, pgi);
-              }
-            }
-          }
-        }
-      }
-      commit = true;
-      return pps;
-    } catch (IOException e) {
-      LOG.error("Unable to get db privileges for user", e);
-      throw new MetaException("Unable to get db privileges for user, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public PrincipalPrivilegeSet getTablePrivilegeSet(String dbName, String tableName,
-                                                    String userName, List<String> groupNames)
-      throws InvalidObjectException, MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      PrincipalPrivilegeSet pps = new PrincipalPrivilegeSet();
-      Table table = getHBase().getTable(dbName, tableName);
-      List<PrivilegeGrantInfo> pgi;
-      if (table.getPrivileges() != null) {
-        if (table.getPrivileges().getUserPrivileges() != null) {
-          pgi = table.getPrivileges().getUserPrivileges().get(userName);
-          if (pgi != null) {
-            pps.putToUserPrivileges(userName, pgi);
-          }
-        }
-
-        if (table.getPrivileges().getRolePrivileges() != null) {
-          List<String> roles = getHBase().getUserRoles(userName);
-          if (roles != null) {
-            for (String role : roles) {
-              pgi = table.getPrivileges().getRolePrivileges().get(role);
-              if (pgi != null) {
-                pps.putToRolePrivileges(role, pgi);
-              }
-            }
-          }
-        }
-      }
-      commit = true;
-      return pps;
-    } catch (IOException e) {
-      LOG.error("Unable to get db privileges for user", e);
-      throw new MetaException("Unable to get db privileges for user, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public PrincipalPrivilegeSet getPartitionPrivilegeSet(String dbName, String tableName,
-                                                        String partition, String userName,
-                                                        List<String> groupNames) throws
-      InvalidObjectException, MetaException {
-    // We don't support partition privileges
-    return null;
-  }
-
-  @Override
-  public PrincipalPrivilegeSet getColumnPrivilegeSet(String dbName, String tableName,
-                                                     String partitionName, String columnName,
-                                                     String userName,
-                                                     List<String> groupNames) throws
-      InvalidObjectException, MetaException {
-    // We don't support column level privileges
-    return null;
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalGlobalGrants(String principalName,
-                                                             PrincipalType principalType) {
-    List<PrivilegeGrantInfo> grants;
-    List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      PrincipalPrivilegeSet pps = getHBase().getGlobalPrivs();
-      if (pps == null) return privileges;
-      Map<String, List<PrivilegeGrantInfo>> map;
-      switch (principalType) {
-        case USER:
-          map = pps.getUserPrivileges();
-          break;
-
-        case ROLE:
-          map = pps.getRolePrivileges();
-          break;
-
-        default:
-          throw new RuntimeException("Unknown or unsupported principal type " +
-              principalType.toString());
-      }
-      if (map == null) return privileges;
-      grants = map.get(principalName);
-
-      if (grants == null || grants.size() == 0) return privileges;
-      for (PrivilegeGrantInfo pgi : grants) {
-        privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.GLOBAL, null,
-            null, null, null), principalName, principalType, pgi));
-      }
-      commit = true;
-      return privileges;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalDBGrants(String principalName,
-                                                         PrincipalType principalType,
-                                                         String dbName) {
-    List<PrivilegeGrantInfo> grants;
-    List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      Database db = getHBase().getDb(dbName);
-      if (db == null) return privileges;
-      PrincipalPrivilegeSet pps = db.getPrivileges();
-      if (pps == null) return privileges;
-      Map<String, List<PrivilegeGrantInfo>> map;
-      switch (principalType) {
-        case USER:
-          map = pps.getUserPrivileges();
-          break;
-
-        case ROLE:
-          map = pps.getRolePrivileges();
-          break;
-
-        default:
-          throw new RuntimeException("Unknown or unsupported principal type " +
-              principalType.toString());
-      }
-      if (map == null) return privileges;
-      grants = map.get(principalName);
-
-      if (grants == null || grants.size() == 0) return privileges;
-      for (PrivilegeGrantInfo pgi : grants) {
-        privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.DATABASE, dbName,
-         null, null, null), principalName, principalType, pgi));
-      }
-      commit = true;
-      return privileges;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listAllTableGrants(String principalName,
-                                                      PrincipalType principalType,
-                                                      String dbName,
-                                                      String tableName) {
-    List<PrivilegeGrantInfo> grants;
-    List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      Table table = getHBase().getTable(dbName, tableName);
-      if (table == null) return privileges;
-      PrincipalPrivilegeSet pps = table.getPrivileges();
-      if (pps == null) return privileges;
-      Map<String, List<PrivilegeGrantInfo>> map;
-      switch (principalType) {
-        case USER:
-          map = pps.getUserPrivileges();
-          break;
-
-        case ROLE:
-          map = pps.getRolePrivileges();
-          break;
-
-        default:
-          throw new RuntimeException("Unknown or unsupported principal type " +
-              principalType.toString());
-      }
-      if (map == null) return privileges;
-      grants = map.get(principalName);
-
-      if (grants == null || grants.size() == 0) return privileges;
-      for (PrivilegeGrantInfo pgi : grants) {
-        privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.TABLE, dbName,
-            tableName, null, null), principalName, principalType, pgi));
-      }
-      commit = true;
-      return privileges;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalPartitionGrants(String principalName,
-                                                                PrincipalType principalType,
-                                                                String dbName,
-                                                                String tableName,
-                                                                List<String> partValues,
-                                                                String partName) {
-    // We don't support partition grants
-    return new ArrayList<HiveObjectPrivilege>();
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalTableColumnGrants(String principalName,
-                                                                  PrincipalType principalType,
-                                                                  String dbName, String tableName,
-                                                                  String columnName) {
-    // We don't support column grants
-    return new ArrayList<HiveObjectPrivilege>();
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalPartitionColumnGrants(String principalName,
-                                                                      PrincipalType principalType,
-                                                                      String dbName,
-                                                                      String tableName,
-                                                                      List<String> partVals,
-                                                                      String partName,
-                                                                      String columnName) {
-    // We don't support column grants
-    return new ArrayList<HiveObjectPrivilege>();
-  }
-
-  @Override
-  public boolean grantPrivileges(PrivilegeBag privileges)
-      throws InvalidObjectException, MetaException, NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      for (HiveObjectPrivilege priv : privileges.getPrivileges()) {
-        // Locate the right object to deal with
-        PrivilegeInfo privilegeInfo = findPrivilegeToGrantOrRevoke(priv);
-
-        // Now, let's see if we've already got this privilege
-        for (PrivilegeGrantInfo info : privilegeInfo.grants) {
-          if (info.getPrivilege().equals(priv.getGrantInfo().getPrivilege())) {
-            throw new InvalidObjectException(priv.getPrincipalName() + " already has " +
-                priv.getGrantInfo().getPrivilege() + " on " + privilegeInfo.typeErrMsg);
-          }
-        }
-        privilegeInfo.grants.add(priv.getGrantInfo());
-
-        writeBackGrantOrRevoke(priv, privilegeInfo);
-      }
-      commit = true;
-      return true;
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) throws
-      InvalidObjectException, MetaException, NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      for (HiveObjectPrivilege priv : privileges.getPrivileges()) {
-        PrivilegeInfo privilegeInfo = findPrivilegeToGrantOrRevoke(priv);
-
-        for (int i = 0; i < privilegeInfo.grants.size(); i++) {
-          if (privilegeInfo.grants.get(i).getPrivilege().equals(
-              priv.getGrantInfo().getPrivilege())) {
-            if (grantOption) privilegeInfo.grants.get(i).setGrantOption(false);
-            else privilegeInfo.grants.remove(i);
-            break;
-          }
-        }
-        writeBackGrantOrRevoke(priv, privilegeInfo);
-      }
-      commit = true;
-      return true;
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  private static class PrivilegeInfo {
-    Database db;
-    Table table;
-    List<PrivilegeGrantInfo> grants;
-    String typeErrMsg;
-    PrincipalPrivilegeSet privSet;
-  }
-
-  private PrivilegeInfo findPrivilegeToGrantOrRevoke(HiveObjectPrivilege privilege)
-      throws MetaException, NoSuchObjectException, InvalidObjectException {
-    PrivilegeInfo result = new PrivilegeInfo();
-    switch (privilege.getHiveObject().getObjectType()) {
-      case GLOBAL:
-        try {
-          result.privSet = createOnNull(getHBase().getGlobalPrivs());
-        } catch (IOException e) {
-          LOG.error("Unable to fetch global privileges", e);
-          throw new MetaException("Unable to fetch global privileges, " + e.getMessage());
-        }
-        result.typeErrMsg = "global";
-        break;
-
-      case DATABASE:
-        result.db = getDatabase(privilege.getHiveObject().getDbName());
-        result.typeErrMsg = "database " + result.db.getName();
-        result.privSet = createOnNull(result.db.getPrivileges());
-        break;
-
-      case TABLE:
-        result.table = getTable(privilege.getHiveObject().getDbName(),
-            privilege.getHiveObject().getObjectName());
-        result.typeErrMsg = "table " + result.table.getTableName();
-        result.privSet = createOnNull(result.table.getPrivileges());
-        break;
-
-      case PARTITION:
-      case COLUMN:
-        throw new RuntimeException("HBase metastore does not support partition or column " +
-            "permissions");
-
-      default:
-        throw new RuntimeException("Woah bad, unknown object type " +
-            privilege.getHiveObject().getObjectType());
-    }
-
-    // Locate the right PrivilegeGrantInfo
-    Map<String, List<PrivilegeGrantInfo>> grantInfos;
-    switch (privilege.getPrincipalType()) {
-      case USER:
-        grantInfos = result.privSet.getUserPrivileges();
-        result.typeErrMsg = "user";
-        break;
-
-      case GROUP:
-        throw new RuntimeException("HBase metastore does not support group permissions");
-
-      case ROLE:
-        grantInfos = result.privSet.getRolePrivileges();
-        result.typeErrMsg = "role";
-        break;
-
-      default:
-        throw new RuntimeException("Woah bad, unknown principal type " +
-            privilege.getPrincipalType());
-    }
-
-    // Find the requested name in the grantInfo
-    result.grants = grantInfos.get(privilege.getPrincipalName());
-    if (result.grants == null) {
-      // Means we don't have any grants for this user yet.
-      result.grants = new ArrayList<PrivilegeGrantInfo>();
-      grantInfos.put(privilege.getPrincipalName(), result.grants);
-    }
-    return result;
-  }
-
-  private PrincipalPrivilegeSet createOnNull(PrincipalPrivilegeSet pps) {
-    // If this is the first time a user has been granted a privilege set will be null.
-    if (pps == null) {
-      pps = new PrincipalPrivilegeSet();
-    }
-    if (pps.getUserPrivileges() == null) {
-      pps.setUserPrivileges(new HashMap<String, List<PrivilegeGrantInfo>>());
-    }
-    if (pps.getRolePrivileges() == null) {
-      pps.setRolePrivileges(new HashMap<String, List<PrivilegeGrantInfo>>());
-    }
-    return pps;
-  }
-
-  private void writeBackGrantOrRevoke(HiveObjectPrivilege priv, PrivilegeInfo pi)
-      throws MetaException, NoSuchObjectException, InvalidObjectException {
-    // Now write it back
-    switch (priv.getHiveObject().getObjectType()) {
-      case GLOBAL:
-        try {
-          getHBase().putGlobalPrivs(pi.privSet);
-        } catch (IOException e) {
-          LOG.error("Unable to write global privileges", e);
-          throw new MetaException("Unable to write global privileges, " + e.getMessage());
-        }
-        break;
-
-      case DATABASE:
-        pi.db.setPrivileges(pi.privSet);
-        alterDatabase(pi.db.getName(), pi.db);
-        break;
-
-      case TABLE:
-        pi.table.setPrivileges(pi.privSet);
-        alterTable(pi.table.getDbName(), pi.table.getTableName(), pi.table);
-        break;
-
-      default:
-        throw new RuntimeException("Dude, you missed the second switch!");
-    }
-  }
-
-  @Override
-  public Role getRole(String roleName) throws NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      Role role = getHBase().getRole(roleName);
-      if (role == null) {
-        throw new NoSuchObjectException("Unable to find role " + roleName);
-      }
-      commit = true;
-      return role;
-    } catch (IOException e) {
-      LOG.error("Unable to get role", e);
-      throw new NoSuchObjectException("Error reading table " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<String> listRoleNames() {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Role> roles = getHBase().scanRoles();
-      List<String> roleNames = new ArrayList<String>(roles.size());
-      for (Role role : roles) roleNames.add(role.getRoleName());
-      commit = true;
-      return roleNames;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<Role> listRoles(String principalName, PrincipalType principalType) {
-    List<Role> roles = new ArrayList<Role>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      try {
-        roles.addAll(getHBase().getPrincipalDirectRoles(principalName, principalType));
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
-      // Add the public role if this is a user
-      if (principalType == PrincipalType.USER) {
-        roles.add(new Role(HiveMetaStore.PUBLIC, 0, null));
-      }
-      commit = true;
-      return roles;
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<RolePrincipalGrant> listRolesWithGrants(String principalName,
-                                                      PrincipalType principalType) {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Role> roles = listRoles(principalName, principalType);
-      List<RolePrincipalGrant> rpgs = new ArrayList<RolePrincipalGrant>(roles.size());
-      for (Role role : roles) {
-        HbaseMetastoreProto.RoleGrantInfoList grants = getHBase().getRolePrincipals(role.getRoleName());
-        if (grants != null) {
-          for (HbaseMetastoreProto.RoleGrantInfo grant : grants.getGrantInfoList()) {
-            if (grant.getPrincipalType() == HBaseUtils.convertPrincipalTypes(principalType) &&
-                grant.getPrincipalName().equals(principalName)) {
-              rpgs.add(new RolePrincipalGrant(role.getRoleName(), principalName, principalType,
-                  grant.getGrantOption(), (int) grant.getAddTime(), grant.getGrantor(),
-                  HBaseUtils.convertPrincipalTypes(grant.getGrantorType())));
-            }
-          }
-        }
-      }
-      commit = true;
-      return rpgs;
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<RolePrincipalGrant> listRoleMembers(String roleName) {
-    boolean commit = false;
-    openTransaction();
-    try {
-      HbaseMetastoreProto.RoleGrantInfoList gil = getHBase().getRolePrincipals(roleName);
-      List<RolePrincipalGrant> roleMaps = new ArrayList<RolePrincipalGrant>(gil.getGrantInfoList().size());
-      for (HbaseMetastoreProto.RoleGrantInfo giw : gil.getGrantInfoList()) {
-        roleMaps.add(new RolePrincipalGrant(roleName, giw.getPrincipalName(),
-            HBaseUtils.convertPrincipalTypes(giw.getPrincipalType()),
-            giw.getGrantOption(), (int)giw.getAddTime(), giw.getGrantor(),
-            HBaseUtils.convertPrincipalTypes(giw.getGrantorType())));
-      }
-      commit = true;
-      return roleMaps;
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public Partition getPartitionWithAuth(String dbName, String tblName, List<String> partVals,
-                                        String user_name, List<String> group_names)
-      throws MetaException, NoSuchObjectException, InvalidObjectException {
-    // We don't do authorization checks for partitions.
-    return getPartition(dbName, tblName, partVals);
-  }
-
-  @Override
-  public List<Partition> getPartitionsWithAuth(String dbName, String tblName, short maxParts,
-                                               String userName, List<String> groupNames)
-      throws MetaException, NoSuchObjectException, InvalidObjectException {
-    // We don't do authorization checks for partitions.
-    return getPartitions(dbName, tblName, maxParts);
-  }
-
-  @Override
-  public List<String> listPartitionNamesPs(String db_name, String tbl_name, List<String> part_vals,
-                                           short max_parts)
-      throws MetaException, NoSuchObjectException {
-    List<Partition> parts =
-        listPartitionsPsWithAuth(db_name, tbl_name, part_vals, max_parts, null, null);
-    List<String> partNames = new ArrayList<String>(parts.size());
-    for (Partition part : parts) {
-      partNames.add(buildExternalPartName(HiveStringUtils.normalizeIdentifier(db_name),
-          HiveStringUtils.normalizeIdentifier(tbl_name), part.getValues()));
-    }
-    return partNames;
-  }
-
-
-  @Override
-  public List<Partition> listPartitionsPsWithAuth(String db_name, String tbl_name,
-                                                  List<String> part_vals, short max_parts,
-                                                  String userName, List<String> groupNames)
-      throws MetaException, NoSuchObjectException {
-    // We don't handle auth info with partitions
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Partition> parts = getHBase().scanPartitions(HiveStringUtils.normalizeIdentifier(db_name),
-          HiveStringUtils.normalizeIdentifier(tbl_name), part_vals, max_parts);
-      commit = true;
-      return parts;
-    } catch (IOException e) {
-      LOG.error("Unable to list partition names", e);
-      throw new MetaException("Failed to list part names, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean updateTableColumnStatistics(ColumnStatistics colStats) throws
-      NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      //update table properties
-      List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
-      List<String> colNames = new ArrayList<>();
-      for (ColumnStatisticsObj statsObj:statsObjs) {
-        colNames.add(statsObj.getColName());
-      }
-      String dbName = colStats.getStatsDesc().getDbName();
-      String tableName = colStats.getStatsDesc().getTableName();
-      Table newTable = getTable(dbName, tableName);
-      Table newTableCopy = newTable.deepCopy();
-      StatsSetupConst.setColumnStatsState(newTableCopy.getParameters(), colNames);
-      getHBase().replaceTable(newTable, newTableCopy);
-
-      getHBase().updateStatistics(colStats.getStatsDesc().getDbName(),
-          colStats.getStatsDesc().getTableName(), null, colStats);
-
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to update column statistics", e);
-      throw new MetaException("Failed to update column statistics, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean updatePartitionColumnStatistics(ColumnStatistics colStats,
-                                                 List<String> partVals) throws
-      NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      // update partition properties
-      String db_name = colStats.getStatsDesc().getDbName();
-      String tbl_name = colStats.getStatsDesc().getTableName();
-      Partition oldPart = getHBase().getPartition(db_name, tbl_name, partVals);
-      Partition new_partCopy = oldPart.deepCopy();
-      List<String> colNames = new ArrayList<>();
-      List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
-      for (ColumnStatisticsObj statsObj : statsObjs) {
-        colNames.add(statsObj.getColName());
-      }
-      StatsSetupConst.setColumnStatsState(new_partCopy.getParameters(), colNames);
-      getHBase().replacePartition(oldPart, new_partCopy,
-          HBaseUtils.getPartitionKeyTypes(getTable(db_name, tbl_name).getPartitionKeys()));
-
-      getHBase().updateStatistics(colStats.getStatsDesc().getDbName(),
-          colStats.getStatsDesc().getTableName(), partVals, colStats);
-      // We need to invalidate aggregates that include this partition
-      getHBase().getStatsCache().invalidate(colStats.getStatsDesc().getDbName(),
-          colStats.getStatsDesc().getTableName(), colStats.getStatsDesc().getPartName());
-
-      commit = true;
-      return true;
-    } catch (IOException e) {
-      LOG.error("Unable to update column statistics", e);
-      throw new MetaException("Failed to update column statistics, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public ColumnStatistics getTableColumnStatistics(String dbName, String tableName,
-                                                   List<String> colName) throws MetaException,
-      NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      ColumnStatistics cs = getHBase().getTableStatistics(dbName, tableName, colName);
-      commit = true;
-      return cs;
-    } catch (IOException e) {
-      LOG.error("Unable to fetch column statistics", e);
-      throw new MetaException("Failed to fetch column statistics, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<ColumnStatistics> getPartitionColumnStatistics(String dbName, String tblName,
-      List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException {
-    List<List<String>> partVals = new ArrayList<List<String>>(partNames.size());
-    for (String partName : partNames) {
-      partVals.add(partNameToVals(partName));
-    }
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<ColumnStatistics> cs =
-          getHBase().getPartitionStatistics(dbName, tblName, partNames, partVals, colNames);
-      commit = true;
-      return cs;
-    } catch (IOException e) {
-      LOG.error("Unable to fetch column statistics", e);
-      throw new MetaException("Failed fetching column statistics, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean deletePartitionColumnStatistics(String dbName, String tableName, String partName,
-      List<String> partVals, String colName) throws NoSuchObjectException, MetaException,
-      InvalidObjectException, InvalidInputException {
-    // NOP, stats will be deleted along with the partition when it is dropped.
-    return true;
-  }
-
-  @Override
-  public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName) throws
-      NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
-    // NOP, stats will be deleted along with the table when it is dropped.
-    return true;
-  }
-
-  /**
-   * Return aggregated statistics for each column in the colNames list aggregated over partitions in
-   * the partNames list
-   *
-   */
-  @Override
-  public AggrStats get_aggr_stats_for(String dbName, String tblName, List<String> partNames,
-      List<String> colNames) throws MetaException, NoSuchObjectException {
-    List<List<String>> partVals = new ArrayList<List<String>>(partNames.size());
-    for (String partName : partNames) {
-      partVals.add(partNameToVals(partName));
-    }
-    boolean commit = false;
-    boolean hasAnyStats = false;
-    openTransaction();
-    try {
-      AggrStats aggrStats = new AggrStats();
-      aggrStats.setPartsFound(0);
-      for (String colName : colNames) {
-        try {
-          AggrStats oneCol =
-              getHBase().getStatsCache().get(dbName, tblName, partNames, colName);
-          if (oneCol.getColStatsSize() > 0) {
-            assert oneCol.getColStatsSize() == 1;
-            aggrStats.setPartsFound(oneCol.getPartsFound());
-            aggrStats.addToColStats(oneCol.getColStats().get(0));
-            hasAnyStats = true;
-          }
-        } catch (CacheLoader.InvalidCacheLoadException e) {
-          LOG.debug("Found no stats for column " + colName);
-          // This means we have no stats at all for this column for these partitions, so just
-          // move on.
-        }
-      }
-      commit = true;
-      if (!hasAnyStats) {
-        // Set the required field.
-        aggrStats.setColStats(new ArrayList<ColumnStatisticsObj>());
-      }
-      return aggrStats;
-    } catch (IOException e) {
-      LOG.error("Unable to fetch aggregate column statistics", e);
-      throw new MetaException("Failed fetching aggregate column statistics, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public long cleanupEvents() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public boolean addToken(String tokenIdentifier, String delegationToken) {
-    boolean commit = false;
-    openTransaction();
-    try {
-      getHBase().putDelegationToken(tokenIdentifier, delegationToken);
-      commit = true;
-      return commit; // See HIVE-11302, for now always returning true
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean removeToken(String tokenIdentifier) {
-    boolean commit = false;
-    openTransaction();
-    try {
-      getHBase().deleteDelegationToken(tokenIdentifier);
-      commit = true;
-      return commit; // See HIVE-11302, for now always returning true
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public String getToken(String tokenIdentifier) {
-    boolean commit = false;
-    openTransaction();
-    try {
-      String token = getHBase().getDelegationToken(tokenIdentifier);
-      commit = true;
-      return token;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<String> getAllTokenIdentifiers() {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<String> ids = getHBase().scanDelegationTokenIdentifiers();
-      commit = true;
-      return ids;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public int addMasterKey(String key) throws MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      long seq = getHBase().getNextSequence(HBaseReadWrite.MASTER_KEY_SEQUENCE);
-      getHBase().putMasterKey((int) seq, key);
-      commit = true;
-      return (int)seq;
-    } catch (IOException e) {
-      LOG.error("Unable to add master key", e);
-      throw new MetaException("Failed adding master key, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public void updateMasterKey(Integer seqNo, String key) throws NoSuchObjectException,
-      MetaException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      if (getHBase().getMasterKey(seqNo) == null) {
-        throw new NoSuchObjectException("No key found with keyId: " + seqNo);
-      }
-      getHBase().putMasterKey(seqNo, key);
-      commit = true;
-    } catch (IOException e) {
-      LOG.error("Unable to update master key", e);
-      throw new MetaException("Failed updating master key, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public boolean removeMasterKey(Integer keySeq) {
-    boolean commit = false;
-    openTransaction();
-    try {
-      getHBase().deleteMasterKey(keySeq);
-      commit = true;
-      return true;  // See HIVE-11302, for now always returning true
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public String[] getMasterKeys() {
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<String> keys = getHBase().scanMasterKeys();
-      commit = true;
-      return keys.toArray(new String[keys.size()]);
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public void verifySchema() throws MetaException {
-
-  }
-
-  @Override
-  public String getMetaStoreSchemaVersion() throws MetaException {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public void setMetaStoreSchemaVersion(String version, String comment) throws MetaException {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public void dropPartitions(String dbName, String tblName, List<String> partNames) throws
-      MetaException, NoSuchObjectException {
-    boolean commit = false;
-    openTransaction();
-    try {
-      for (String partName : partNames) {
-        dropPartition(dbName, tblName, partNameToVals(partName));
-      }
-      commit = true;
-    } catch (Exception e) {
-      LOG.error("Unable to drop partitions", e);
-      throw new NoSuchObjectException("Failure dropping partitions, " + e.getMessage());
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalDBGrantsAll(String principalName,
-                                                            PrincipalType principalType) {
-    List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Database> dbs = getHBase().scanDatabases(null);
-      for (Database db : dbs) {
-        List<PrivilegeGrantInfo> grants;
-
-        PrincipalPrivilegeSet pps = db.getPrivileges();
-        if (pps == null) continue;
-        Map<String, List<PrivilegeGrantInfo>> map;
-        switch (principalType) {
-          case USER:
-            map = pps.getUserPrivileges();
-            break;
-
-          case ROLE:
-            map = pps.getRolePrivileges();
-            break;
-
-          default:
-            throw new RuntimeException("Unknown or unsupported principal type " +
-                principalType.toString());
-        }
-
-        if (map == null) continue;
-        grants = map.get(principalName);
-        if (grants == null || grants.size() == 0) continue;
-        for (PrivilegeGrantInfo pgi : grants) {
-          privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.DATABASE,
-              db.getName(), null, null, null), principalName, principalType, pgi));
-        }
-      }
-      commit = true;
-      return privileges;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalTableGrantsAll(String principalName,
-                                                               PrincipalType principalType) {
-    List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      List<Table> tables = getHBase().scanTables(null, null);
-      for (Table table : tables) {
-        List<PrivilegeGrantInfo> grants;
-
-        PrincipalPrivilegeSet pps = table.getPrivileges();
-        if (pps == null) continue;
-        Map<String, List<PrivilegeGrantInfo>> map;
-        switch (principalType) {
-          case USER:
-            map = pps.getUserPrivileges();
-            break;
-
-          case ROLE:
-            map = pps.getRolePrivileges();
-            break;
-
-          default:
-            throw new RuntimeException("Unknown or unsupported principal type " +
-                principalType.toString());
-        }
-
-        if (map == null) continue;
-        grants = map.get(principalName);
-        if (grants == null || grants.size() == 0) continue;
-        for (PrivilegeGrantInfo pgi : grants) {
-          privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.TABLE,
-              table.getDbName(), table.getTableName(), null, null), principalName, principalType,
-              pgi));
-        }
-      }
-      commit = true;
-      return privileges;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalPartitionGrantsAll(String principalName,
-                                                                   PrincipalType principalType) {
-    return new ArrayList<HiveObjectPrivilege>();
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalTableColumnGrantsAll(String principalName,
-                                                                     PrincipalType principalType) {
-    return new ArrayList<HiveObjectPrivilege>();
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listPrincipalPartitionColumnGrantsAll(String principalName,
-                                                                         PrincipalType principalType) {
-    return new ArrayList<HiveObjectPrivilege>();
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listGlobalGrantsAll() {
-    List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-    boolean commit = false;
-    openTransaction();
-    try {
-      PrincipalPrivilegeSet pps = getHBase().getGlobalPrivs();
-      if (pps != null) {
-        for (Map.Entry<String, List<PrivilegeGrantInfo>> e : pps.getUserPrivileges().entrySet()) {
-          for (PrivilegeGrantInfo pgi : e.getValue()) {
-            privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.GLOBAL, null,
-                null, null, null), e.getKey(), PrincipalType.USER, pgi));
-          }
-        }
-        for (Map.Entry<String, List<PrivilegeGrantInfo>> e : pps.getRolePrivileges().entrySet()) {
-          for (PrivilegeGrantInfo pgi : e.getValue()) {
-            privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.GLOBAL, null,
-                null, null, null), e.getKey(), PrincipalType.ROLE, pgi));
-          }
-        }
-      }
-      commit = true;
-      return privileges;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    } finally {
-      commitOrRoleBack(commit);
-    }
-  }
-
-  @Override
-  public List<HiveObjectPrivilege> listDBGrantsAll(String dbName) {
-    List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-    boolean commit = false;
-   

<TRUNCATED>

[15/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
deleted file mode 100644
index 2cc1373..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
+++ /dev/null
@@ -1,1801 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.HiveObjectType;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.ResourceType;
-import org.apache.hadoop.hive.metastore.api.ResourceUri;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Integration tests with HBase Mini-cluster for HBaseStore
- */
-public class TestHBaseStoreIntegration extends HBaseIntegrationTests {
-
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreIntegration.class.getName());
-
-  @Rule public ExpectedException thrown = ExpectedException.none();
-
-  @BeforeClass
-  public static void startup() throws Exception {
-    HBaseIntegrationTests.startMiniCluster();
-  }
-
-  @AfterClass
-  public static void shutdown() throws Exception {
-    HBaseIntegrationTests.shutdownMiniCluster();
-  }
-
-  @Before
-  public void setup() throws IOException {
-    setupConnection();
-    setupHBaseStore();
-  }
-
-  @Test
-  public void createDb() throws Exception {
-    String dbname = "mydb";
-    Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
-    store.createDatabase(db);
-
-    Database d = store.getDatabase("mydb");
-    Assert.assertEquals(dbname, d.getName());
-    Assert.assertEquals("no description", d.getDescription());
-    Assert.assertEquals("file:///tmp", d.getLocationUri());
-  }
-
-  @Test
-  public void dropDb() throws Exception {
-    String dbname = "anotherdb";
-    Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
-    store.createDatabase(db);
-
-    Database d = store.getDatabase(dbname);
-    Assert.assertNotNull(d);
-
-    store.dropDatabase(dbname);
-    thrown.expect(NoSuchObjectException.class);
-    store.getDatabase(dbname);
-  }
-
-  @Test
-  public void getAllDbs() throws Exception {
-    String[] dbNames = new String[3];
-    for (int i = 0; i < dbNames.length; i++) {
-      dbNames[i] = "db" + i;
-      Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters);
-      store.createDatabase(db);
-    }
-
-    List<String> dbs = store.getAllDatabases();
-    Assert.assertEquals(3, dbs.size());
-    String[] namesFromStore = dbs.toArray(new String[3]);
-    Arrays.sort(namesFromStore);
-    Assert.assertArrayEquals(dbNames, namesFromStore);
-  }
-
-  @Test
-  public void getDbsRegex() throws Exception {
-    String[] dbNames = new String[3];
-    for (int i = 0; i < dbNames.length; i++) {
-      dbNames[i] = "db" + i;
-      Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters);
-      store.createDatabase(db);
-    }
-
-    List<String> dbs = store.getDatabases("db1|db2");
-    Assert.assertEquals(2, dbs.size());
-    String[] namesFromStore = dbs.toArray(new String[2]);
-    Arrays.sort(namesFromStore);
-    Assert.assertArrayEquals(Arrays.copyOfRange(dbNames, 1, 3), namesFromStore);
-
-    dbs = store.getDatabases("db*");
-    Assert.assertEquals(3, dbs.size());
-    namesFromStore = dbs.toArray(new String[3]);
-    Arrays.sort(namesFromStore);
-    Assert.assertArrayEquals(dbNames, namesFromStore);
-  }
-
-  @Test
-  public void getFuncsRegex() throws Exception {
-    String dbname = "default";
-    int now = (int)(System.currentTimeMillis()/1000);
-    String[] funcNames = new String[3];
-    for (int i = 0; i < funcNames.length; i++) {
-      funcNames[i] = "func" + i;
-      store.createFunction(new Function(funcNames[i], dbname, "o.a.h.h.myfunc", "me",
-                                        PrincipalType.USER, now, FunctionType.JAVA,
-                                        Arrays.asList(new ResourceUri(ResourceType.JAR,
-                                        "file:/tmp/somewhere"))));
-    }
-
-    List<String> funcs = store.getFunctions(dbname, "func1|func2");
-    Assert.assertEquals(2, funcs.size());
-    String[] namesFromStore = funcs.toArray(new String[2]);
-    Arrays.sort(namesFromStore);
-    Assert.assertArrayEquals(Arrays.copyOfRange(funcNames, 1, 3), namesFromStore);
-
-    funcs = store.getFunctions(dbname, "func*");
-    Assert.assertEquals(3, funcs.size());
-    namesFromStore = funcs.toArray(new String[3]);
-    Arrays.sort(namesFromStore);
-    Assert.assertArrayEquals(funcNames, namesFromStore);
-
-    funcs = store.getFunctions("nosuchdb", "func*");
-    Assert.assertEquals(0, funcs.size());
-  }
-
-  @Test
-  public void createTable() throws Exception {
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table("mytable", "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    Table t = store.getTable("default", "mytable");
-    Assert.assertEquals(1, t.getSd().getColsSize());
-    Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
-    Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp", t.getSd().getLocation());
-    Assert.assertEquals("input", t.getSd().getInputFormat());
-    Assert.assertEquals("output", t.getSd().getOutputFormat());
-    Assert.assertEquals("me", t.getOwner());
-    Assert.assertEquals("default", t.getDbName());
-    Assert.assertEquals("mytable", t.getTableName());
-  }
-
-  @Test
-  public void alterTable() throws Exception {
-    String tableName = "alttable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    startTime += 10;
-    table.setLastAccessTime(startTime);
-    LOG.debug("XXX alter table test");
-    store.alterTable("default", tableName, table);
-
-    Table t = store.getTable("default", tableName);
-    LOG.debug("Alter table time " + t.getLastAccessTime());
-    Assert.assertEquals(1, t.getSd().getColsSize());
-    Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
-    Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp", t.getSd().getLocation());
-    Assert.assertEquals("input", t.getSd().getInputFormat());
-    Assert.assertEquals("output", t.getSd().getOutputFormat());
-    Assert.assertEquals("me", t.getOwner());
-    Assert.assertEquals("default", t.getDbName());
-    Assert.assertEquals(tableName, t.getTableName());
-    Assert.assertEquals(startTime, t.getLastAccessTime());
-  }
-
-  @Test
-  public void getAllTables() throws Exception {
-    String dbNames[] = new String[]{"db0", "db1"}; // named to match getAllDbs so we get the
-    // right number of databases in that test.
-    String tableNames[] = new String[]{"curly", "larry", "moe"};
-
-    for (int i = 0; i < dbNames.length; i++) {
-      store.createDatabase(new Database(dbNames[i], "no description", "file:///tmp",
-          emptyParameters));
-    }
-
-    for (int i = 0; i < dbNames.length; i++) {
-      for (int j = 0; j < tableNames.length; j++) {
-        int startTime = (int) (System.currentTimeMillis() / 1000);
-        List<FieldSchema> cols = new ArrayList<FieldSchema>();
-        cols.add(new FieldSchema("col1", "int", "nocomment"));
-        SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-        StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false,
-            0,
-            serde, null, null, emptyParameters);
-        Table table = new Table(tableNames[j], dbNames[i], "me", startTime, startTime, 0, sd,
-            null,
-            emptyParameters, null, null, null);
-        store.createTable(table);
-      }
-    }
-
-    List<String> fetchedNames = store.getAllTables(dbNames[0]);
-    Assert.assertEquals(3, fetchedNames.size());
-    String[] sortedFetchedNames = fetchedNames.toArray(new String[fetchedNames.size()]);
-    Arrays.sort(sortedFetchedNames);
-    Assert.assertArrayEquals(tableNames, sortedFetchedNames);
-
-    List<String> regexNames = store.getTables(dbNames[0], "*y");
-    Assert.assertEquals(2, regexNames.size());
-    String[] sortedRegexNames = regexNames.toArray(new String[regexNames.size()]);
-    Arrays.sort(sortedRegexNames);
-    Assert.assertArrayEquals(Arrays.copyOfRange(tableNames, 0, 2), sortedRegexNames);
-
-    List<Table> fetchedTables = store.getTableObjectsByName(dbNames[1],
-        Arrays.asList(Arrays.copyOfRange(tableNames, 1, 3)));
-    Assert.assertEquals(2, fetchedTables.size());
-    sortedFetchedNames = new String[fetchedTables.size()];
-    for (int i = 0; i < fetchedTables.size(); i++) {
-      sortedFetchedNames[i] = fetchedTables.get(i).getTableName();
-    }
-    Arrays.sort(sortedFetchedNames);
-    Assert.assertArrayEquals(Arrays.copyOfRange(tableNames, 1, 3), sortedFetchedNames);
-  }
-
-  @Test
-  public void dropTable() throws Exception {
-    String tableName = "dtable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    Table t = store.getTable("default", tableName);
-    Assert.assertNotNull(t);
-
-    store.dropTable("default", tableName);
-    Assert.assertNull(store.getTable("default", tableName));
-  }
-
-  @Test
-  public void createPartition() throws Exception {
-    String dbName = "default";
-    String tableName = "myparttable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> vals = new ArrayList<String>();
-    vals.add("fred");
-    StorageDescriptor psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/pc=fred");
-    Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-        emptyParameters);
-    store.addPartition(part);
-
-    Partition p = store.getPartition(dbName, tableName, vals);
-    Assert.assertEquals(1, p.getSd().getColsSize());
-    Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
-    Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
-    Assert.assertEquals("input", p.getSd().getInputFormat());
-    Assert.assertEquals("output", p.getSd().getOutputFormat());
-    Assert.assertEquals(dbName, p.getDbName());
-    Assert.assertEquals(tableName, p.getTableName());
-    Assert.assertEquals(1, p.getValuesSize());
-    Assert.assertEquals("fred", p.getValues().get(0));
-  }
-
-  @Test
-  public void addPartitions() throws Exception {
-    String dbName = "default";
-    String tableName = "addParts";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
-    List<Partition> partitions = new ArrayList<Partition>();
-    for (String val : partVals) {
-      List<String> vals = new ArrayList<String>();
-      vals.add(val);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/pc=" + val);
-      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-          emptyParameters);
-      partitions.add(part);
-    }
-    store.addPartitions(dbName, tableName, partitions);
-
-    List<String> partNames = store.listPartitionNames(dbName, tableName, (short) -1);
-    Assert.assertEquals(5, partNames.size());
-    String[] names = partNames.toArray(new String[partNames.size()]);
-    Arrays.sort(names);
-    String[] canonicalNames = partVals.toArray(new String[partVals.size()]);
-    for (int i = 0; i < canonicalNames.length; i++) canonicalNames[i] = "pc=" + canonicalNames[i];
-    Assert.assertArrayEquals(canonicalNames, names);
-  }
-
-  @Test
-  public void alterPartitions() throws Exception {
-    String dbName = "default";
-    String tableName = "alterParts";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
-    List<Partition> partitions = new ArrayList<Partition>();
-    List<List<String>> allVals = new ArrayList<List<String>>();
-    for (String val : partVals) {
-      List<String> vals = new ArrayList<String>();
-      allVals.add(vals);
-      vals.add(val);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/pc=" + val);
-      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-          emptyParameters);
-      partitions.add(part);
-    }
-    store.addPartitions(dbName, tableName, partitions);
-
-    for (Partition p : partitions) p.setLastAccessTime(startTime + 10);
-    store.alterPartitions(dbName, tableName, allVals, partitions);
-
-    partitions = store.getPartitions(dbName, tableName, -1);
-    for (Partition part : partitions) {
-      Assert.assertEquals(startTime + 10, part.getLastAccessTime());
-    }
-  }
-
-  @Test
-  public void getPartitions() throws Exception {
-    String dbName = "default";
-    String tableName = "manyParts";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
-    for (String val : partVals) {
-      List<String> vals = new ArrayList<String>();
-      vals.add(val);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/pc=" + val);
-      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-          emptyParameters);
-      store.addPartition(part);
-
-      Partition p = store.getPartition(dbName, tableName, vals);
-      Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
-    }
-
-    List<Partition> parts = store.getPartitions(dbName, tableName, -1);
-    Assert.assertEquals(5, parts.size());
-    String[] pv = new String[5];
-    for (int i = 0; i < 5; i++) pv[i] = parts.get(i).getValues().get(0);
-    Arrays.sort(pv);
-    Assert.assertArrayEquals(pv, partVals.toArray(new String[5]));
-  }
-
-  @Test
-  public void listPartitions() throws Exception {
-    String dbName = "default";
-    String tableName = "listParts";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    partCols.add(new FieldSchema("region", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    String[][] partVals = new String[][]{{"today", "north america"}, {"tomorrow", "europe"}};
-    for (String[] pv : partVals) {
-      List<String> vals = new ArrayList<String>();
-      for (String v : pv) vals.add(v);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/pc=" + pv[0] + "/region=" + pv[1]);
-      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-          emptyParameters);
-      store.addPartition(part);
-    }
-
-    List<String> names = store.listPartitionNames(dbName, tableName, (short) -1);
-    Assert.assertEquals(2, names.size());
-    String[] resultNames = names.toArray(new String[names.size()]);
-    Arrays.sort(resultNames);
-    Assert.assertArrayEquals(resultNames, new String[]{"pc=today/region=north america",
-        "pc=tomorrow/region=europe"});
-
-    List<Partition> parts = store.getPartitionsByNames(dbName, tableName, names);
-    Assert.assertArrayEquals(partVals[0], parts.get(0).getValues().toArray(new String[2]));
-    Assert.assertArrayEquals(partVals[1], parts.get(1).getValues().toArray(new String[2]));
-
-    store.dropPartitions(dbName, tableName, names);
-    List<Partition> afterDropParts = store.getPartitions(dbName, tableName, -1);
-    Assert.assertEquals(0, afterDropParts.size());
-  }
-
-  @Test
-  public void listPartitionsWithPs() throws Exception {
-    String dbName = "default";
-    String tableName = "listPartitionsWithPs";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    partCols.add(new FieldSchema("region", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    String[][] partVals = new String[][]{{"today", "north america"}, {"today", "europe"},
-        {"tomorrow", "north america"}, {"tomorrow", "europe"}};
-    for (String[] pv : partVals) {
-      List<String> vals = new ArrayList<String>();
-      for (String v : pv) vals.add(v);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]);
-      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-          emptyParameters);
-      store.addPartition(part);
-    }
-
-    // We only test listPartitionNamesPs since it calls listPartitionsPsWithAuth anyway.
-    // Test the case where we completely specify the partition
-    List<String> partitionNames =
-        store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0]), (short) -1);
-    Assert.assertEquals(1, partitionNames.size());
-    Assert.assertEquals("ds=today/region=north america", partitionNames.get(0));
-
-    // Leave off the last value of the partition
-    partitionNames =
-        store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0][0]), (short)-1);
-    Assert.assertEquals(2, partitionNames.size());
-    String[] names = partitionNames.toArray(new String[partitionNames.size()]);
-    Arrays.sort(names);
-    Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
-        "ds=today/region=north america"}, names);
-
-    // Put a star in the last value of the partition
-    partitionNames =
-        store.listPartitionNamesPs(dbName, tableName, Arrays.asList("today", "*"), (short)-1);
-    Assert.assertEquals(2, partitionNames.size());
-    names = partitionNames.toArray(new String[partitionNames.size()]);
-    Arrays.sort(names);
-    Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
-        "ds=today/region=north america"}, names);
-
-    // Put a star in the first value of the partition
-    partitionNames =
-        store.listPartitionNamesPs(dbName, tableName, Arrays.asList("*", "europe"), (short)-1);
-    Assert.assertEquals(2, partitionNames.size());
-    names = partitionNames.toArray(new String[partitionNames.size()]);
-    Arrays.sort(names);
-    Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
-        "ds=tomorrow/region=europe"}, names);
-  }
-
-
-  @Test
-  public void getPartitionsByFilter() throws Exception {
-    String dbName = "default";
-    String tableName = "getPartitionsByFilter";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    partCols.add(new FieldSchema("region", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    String[][] partVals = new String[][]{{"20010101", "north america"}, {"20010101", "europe"},
-        {"20010102", "north america"}, {"20010102", "europe"}, {"20010103", "north america"}};
-    for (String[] pv : partVals) {
-      List<String> vals = new ArrayList<String>();
-      for (String v : pv) vals.add(v);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]);
-      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-          emptyParameters);
-      store.addPartition(part);
-    }
-
-    // We only test getPartitionsByFilter since it calls same code as getPartitionsByExpr anyway.
-    // Test the case where we completely specify the partition
-    List<Partition> parts = null;
-    parts = store.getPartitionsByFilter(dbName, tableName, "ds > '20010101'", (short) -1);
-    checkPartVals(parts, "[20010102, north america]", "[20010102, europe]",
-        "[20010103, north america]");
-
-    parts = store.getPartitionsByFilter(dbName, tableName, "ds >= '20010102'", (short) -1);
-    checkPartVals(parts, "[20010102, north america]", "[20010102, europe]",
-        "[20010103, north america]");
-
-    parts = store.getPartitionsByFilter(dbName, tableName,
-        "ds >= '20010102' and region = 'europe' ", (short) -1);
-    // filtering on first partition is only implemented as of now, so it will
-    // not filter on region
-    checkPartVals(parts, "[20010102, north america]", "[20010102, europe]",
-        "[20010103, north america]");
-
-    parts = store.getPartitionsByFilter(dbName, tableName,
-        "ds >= '20010101' and ds < '20010102'", (short) -1);
-    checkPartVals(parts,"[20010101, north america]", "[20010101, europe]");
-
-    parts = store.getPartitionsByFilter(dbName, tableName,
-        "ds = '20010102' or ds < '20010103'", (short) -1);
-    checkPartVals(parts, "[20010101, north america]", "[20010101, europe]",
-        "[20010102, north america]", "[20010102, europe]");
-
-    // test conversion to DNF
-    parts = store.getPartitionsByFilter(dbName, tableName,
-        "ds = '20010102' and (ds = '20010102' or region = 'europe')", (short) -1);
-    // filtering on first partition is only implemented as of now, so it will not filter on region
-    checkPartVals(parts, "[20010102, north america]", "[20010102, europe]");
-
-    parts = store.getPartitionsByFilter(dbName, tableName,
-        "region = 'europe'", (short) -1);
-    // filtering on first partition is only implemented as of now, so it will not filter on region
-    checkPartVals(parts, "[20010101, north america]", "[20010101, europe]",
-        "[20010102, north america]", "[20010102, europe]", "[20010103, north america]");
-
-  }
-
-  /**
-   * Check if the given partitions have same values as given partitions value strings
-   * @param parts given partitions
-   * @param expectedPartVals
-   */
-  private void checkPartVals(List<Partition> parts, String ... expectedPartVals) {
-    Assert.assertEquals("number of partitions", expectedPartVals.length, parts.size());
-    Set<String> partValStrings = new TreeSet<String>();
-    for(Partition part : parts) {
-      partValStrings.add(part.getValues().toString());
-    }
-    partValStrings.equals(new TreeSet(Arrays.asList(expectedPartVals)));
-  }
-
-  @Test
-  public void dropPartition() throws Exception {
-    String dbName = "default";
-    String tableName = "myparttable2";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> vals = Arrays.asList("fred");
-    StorageDescriptor psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/pc=fred");
-    Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-        emptyParameters);
-    store.addPartition(part);
-
-    Assert.assertNotNull(store.getPartition(dbName, tableName, vals));
-    store.dropPartition(dbName, tableName, vals);
-    thrown.expect(NoSuchObjectException.class);
-    store.getPartition(dbName, tableName, vals);
-  }
-
-  @Test
-  public void createRole() throws Exception {
-    int now = (int)System.currentTimeMillis()/1000;
-    String roleName = "myrole";
-    store.addRole(roleName, "me");
-
-    Role r = store.getRole(roleName);
-    Assert.assertEquals(roleName, r.getRoleName());
-    Assert.assertEquals("me", r.getOwnerName());
-    Assert.assertTrue(now <= r.getCreateTime());
-  }
-
-  @Test
-  public void dropRole() throws Exception {
-    String roleName = "anotherrole";
-    store.addRole(roleName, "me");
-
-    Role r = store.getRole(roleName);
-    Assert.assertEquals(roleName, r.getRoleName());
-
-    store.removeRole(roleName);
-    thrown.expect(NoSuchObjectException.class);
-    store.getRole(roleName);
-  }
-
-  @Test
-  public void grantRevokeRoles() throws Exception {
-    int now = (int)(System.currentTimeMillis()/1000);
-    String roleName1 = "role1";
-    store.addRole(roleName1, "me");
-    String roleName2 = "role2";
-    store.addRole(roleName2, "me");
-
-    Role role1 = store.getRole(roleName1);
-    Role role2 = store.getRole(roleName2);
-
-    store.grantRole(role1, "fred", PrincipalType.USER, "bob", PrincipalType.USER, false);
-    store.grantRole(role2, roleName1, PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
-    store.grantRole(role2, "fred", PrincipalType.USER, "admin", PrincipalType.ROLE, false);
-
-    List<Role> roles = store.listRoles("fred", PrincipalType.USER);
-    Assert.assertEquals(3, roles.size());
-    boolean sawRole1 = false, sawRole2 = false, sawPublic = false;
-    for (Role role : roles) {
-      if (role.getRoleName().equals(roleName1)) {
-        sawRole1 = true;
-      } else if (role.getRoleName().equals(roleName2)) {
-        sawRole2 = true;
-      } else if (role.getRoleName().equals(HiveMetaStore.PUBLIC)) {
-        sawPublic = true;
-      } else {
-        Assert.fail("Unknown role name " + role.getRoleName());
-      }
-    }
-    Assert.assertTrue(sawRole1 && sawRole2 && sawPublic);
-
-    roles = store.listRoles("fred", PrincipalType.ROLE);
-    Assert.assertEquals(0, roles.size());
-
-    roles = store.listRoles(roleName1, PrincipalType.ROLE);
-    Assert.assertEquals(1, roles.size());
-    Role role = roles.get(0);
-    Assert.assertEquals(roleName2, role.getRoleName());
-
-    // Test listing all members in a role
-    List<RolePrincipalGrant> grants = store.listRoleMembers(roleName1);
-    Assert.assertEquals(1, grants.size());
-    Assert.assertEquals("fred", grants.get(0).getPrincipalName());
-    Assert.assertEquals(PrincipalType.USER, grants.get(0).getPrincipalType());
-    Assert.assertTrue("Expected grant time of " + now + " got " + grants.get(0).getGrantTime(),
-        grants.get(0).getGrantTime() >= now);
-    Assert.assertEquals("bob", grants.get(0).getGrantorName());
-    Assert.assertEquals(PrincipalType.USER, grants.get(0).getGrantorPrincipalType());
-    Assert.assertFalse(grants.get(0).isGrantOption());
-
-    grants = store.listRoleMembers(roleName2);
-    Assert.assertEquals(2, grants.size());
-    boolean sawFred = false;
-    sawRole1 = false;
-    for (RolePrincipalGrant m : grants) {
-      if ("fred".equals(m.getPrincipalName())) sawFred = true;
-      else if (roleName1.equals(m.getPrincipalName())) sawRole1 = true;
-      else Assert.fail("Unexpected principal " + m.getPrincipalName());
-    }
-    Assert.assertTrue(sawFred && sawRole1);
-
-    // Revoke a role with grant option, make sure it just goes to no grant option
-    store.revokeRole(role2, roleName1, PrincipalType.ROLE, true);
-    roles = store.listRoles(roleName1, PrincipalType.ROLE);
-    Assert.assertEquals(1, roles.size());
-    Assert.assertEquals(roleName2, roles.get(0).getRoleName());
-
-    grants = store.listRoleMembers(roleName1);
-    Assert.assertFalse(grants.get(0).isGrantOption());
-
-    // Drop a role, make sure it is properly removed from the map
-    store.removeRole(roleName1);
-    roles = store.listRoles("fred", PrincipalType.USER);
-    Assert.assertEquals(2, roles.size());
-    sawRole2 = sawPublic = false;
-    for (Role m : roles) {
-      if (m.getRoleName().equals(roleName2)) sawRole2 = true;
-      else if (m.getRoleName().equals(HiveMetaStore.PUBLIC)) sawPublic = true;
-      else Assert.fail("Unknown role " + m.getRoleName());
-    }
-    Assert.assertTrue(sawRole2 && sawPublic);
-    roles = store.listRoles(roleName1, PrincipalType.ROLE);
-    Assert.assertEquals(0, roles.size());
-
-    // Revoke a role without grant option, make sure it goes away
-    store.revokeRole(role2, "fred", PrincipalType.USER, false);
-    roles = store.listRoles("fred", PrincipalType.USER);
-    Assert.assertEquals(1, roles.size());
-    Assert.assertEquals(HiveMetaStore.PUBLIC, roles.get(0).getRoleName());
-  }
-
-  @Test
-  public void userToRoleMap() throws Exception {
-    String roleName1 = "utrm1";
-    store.addRole(roleName1, "me");
-    String roleName2 = "utrm2";
-    store.addRole(roleName2, "me");
-    String user1 = "wilma";
-    String user2 = "betty";
-
-    Role role1 = store.getRole(roleName1);
-    Role role2 = store.getRole(roleName2);
-
-    store.grantRole(role1, user1, PrincipalType.USER, "bob", PrincipalType.USER, false);
-    store.grantRole(role1, roleName2, PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
-
-    List<String> roles = HBaseReadWrite.getInstance().getUserRoles(user1);
-    Assert.assertEquals(2, roles.size());
-    String[] roleNames = roles.toArray(new String[roles.size()]);
-    Arrays.sort(roleNames);
-    Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames);
-
-    store.grantRole(role2, user1, PrincipalType.USER, "admin", PrincipalType.ROLE, false);
-    store.grantRole(role1, user2, PrincipalType.USER, "bob", PrincipalType.USER, false);
-
-    HBaseReadWrite.setConf(conf);
-    roles = HBaseReadWrite.getInstance().getUserRoles(user2);
-    Assert.assertEquals(2, roles.size());
-    roleNames = roles.toArray(new String[roles.size()]);
-    Arrays.sort(roleNames);
-    Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames);
-
-    store.revokeRole(role1, roleName2, PrincipalType.ROLE, false);
-
-    // user1 should still have both roles since she was granted into role1 specifically.  user2
-    // should only have role2 now since role2 was revoked from role1.
-    roles = HBaseReadWrite.getInstance().getUserRoles(user1);
-    Assert.assertEquals(2, roles.size());
-    roleNames = roles.toArray(new String[roles.size()]);
-    Arrays.sort(roleNames);
-    Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames);
-
-    roles = HBaseReadWrite.getInstance().getUserRoles(user2);
-    Assert.assertEquals(1, roles.size());
-    Assert.assertEquals(roleName1, roles.get(0));
-  }
-
-  @Test
-  public void userToRoleMapOnDrop() throws Exception {
-    String roleName1 = "utrmod1";
-    store.addRole(roleName1, "me");
-    String roleName2 = "utrmod2";
-    store.addRole(roleName2, "me");
-    String user1 = "pebbles";
-    String user2 = "bam-bam";
-
-    Role role1 = store.getRole(roleName1);
-    Role role2 = store.getRole(roleName2);
-
-    store.grantRole(role1, user1, PrincipalType.USER, "bob", PrincipalType.USER, false);
-    store.grantRole(role1, roleName2, PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
-    store.grantRole(role1, user2, PrincipalType.USER, "bob", PrincipalType.USER, false);
-
-    List<String> roles = HBaseReadWrite.getInstance().getUserRoles(user2);
-    Assert.assertEquals(2, roles.size());
-    String[] roleNames = roles.toArray(new String[roles.size()]);
-    Arrays.sort(roleNames);
-    Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames);
-
-    store.removeRole(roleName2);
-
-    HBaseReadWrite.setConf(conf);
-    roles = HBaseReadWrite.getInstance().getUserRoles(user1);
-    Assert.assertEquals(1, roles.size());
-    Assert.assertEquals(roleName1, roles.get(0));
-
-    roles = HBaseReadWrite.getInstance().getUserRoles(user2);
-    Assert.assertEquals(1, roles.size());
-    Assert.assertEquals(roleName1, roles.get(0));
-  }
-
-  @Test
-  public void grantRevokeGlobalPrivileges() throws Exception {
-    doGrantRevoke(HiveObjectType.GLOBAL, null, null, new String[] {"grpg1", "grpg2"},
-        new String[] {"bugs", "elmer", "daphy", "wiley"});
-  }
-
-  @Test
-  public void grantRevokeDbPrivileges() throws Exception {
-    String dbName = "grdbp_db";
-    try {
-      Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters);
-      store.createDatabase(db);
-      doGrantRevoke(HiveObjectType.DATABASE, dbName, null,
-          new String[] {"grdbp_role1", "grdbp_role2"},
-          new String[] {"fred", "barney", "wilma", "betty"});
-    } finally {
-      store.dropDatabase(dbName);
-    }
-  }
-
-  @Test
-  public void grantRevokeTablePrivileges() throws Exception {
-    String dbName = "grtp_db";
-    String tableName = "grtp_table";
-    try {
-      Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters);
-      store.createDatabase(db);
-      int startTime = (int)(System.currentTimeMillis() / 1000);
-      List<FieldSchema> cols = new ArrayList<FieldSchema>();
-      cols.add(new FieldSchema("col1", "int", "nocomment"));
-      SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-      StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-          serde, null, null, emptyParameters);
-      Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null,
-          emptyParameters, null, null, null);
-      store.createTable(table);
-      doGrantRevoke(HiveObjectType.TABLE, dbName, tableName,
-          new String[] {"grtp_role1", "grtp_role2"},
-          new String[] {"batman", "robin", "superman", "wonderwoman"});
-
-    } finally {
-      if (store.getTable(dbName, tableName) != null) store.dropTable(dbName, tableName);
-      store.dropDatabase(dbName);
-    }
-  }
-
-  private void doGrantRevoke(HiveObjectType objectType, String dbName, String tableName,
-                             String[] roleNames, String[] userNames)
-      throws Exception {
-    store.addRole(roleNames[0], "me");
-    store.addRole(roleNames[1], "me");
-    int now = (int)(System.currentTimeMillis() / 1000);
-
-    Role role1 = store.getRole(roleNames[0]);
-    Role role2 = store.getRole(roleNames[1]);
-    store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false);
-    store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
-    store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false);
-
-    List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-    HiveObjectRef hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
-    PrivilegeGrantInfo grantInfo =
-        new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false);
-    HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER,
-        grantInfo);
-    privileges.add(hop);
-
-    hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
-    grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
-    hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
-    privileges.add(hop);
-
-    hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
-    grantInfo = new PrivilegeGrantInfo("exec", now, "me", PrincipalType.USER, false);
-    hop = new HiveObjectPrivilege(hiveObjRef, roleNames[1], PrincipalType.ROLE, grantInfo);
-    privileges.add(hop);
-
-    hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
-    grantInfo = new PrivilegeGrantInfo("create", now, "me", PrincipalType.USER, true);
-    hop = new HiveObjectPrivilege(hiveObjRef, userNames[2], PrincipalType.USER, grantInfo);
-    privileges.add(hop);
-
-    hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
-    grantInfo = new PrivilegeGrantInfo("create2", now, "me", PrincipalType.USER, true);
-    hop = new HiveObjectPrivilege(hiveObjRef, userNames[2], PrincipalType.USER, grantInfo);
-    privileges.add(hop);
-
-    PrivilegeBag pBag = new PrivilegeBag(privileges);
-    store.grantPrivileges(pBag);
-
-    PrincipalPrivilegeSet pps = getPPS(objectType, dbName, tableName, userNames[0]);
-
-    Assert.assertEquals(1, pps.getUserPrivilegesSize());
-    Assert.assertEquals(1, pps.getUserPrivileges().get(userNames[0]).size());
-    grantInfo = pps.getUserPrivileges().get(userNames[0]).get(0);
-    Assert.assertEquals("read", grantInfo.getPrivilege());
-    Assert.assertTrue(now <= grantInfo.getCreateTime());
-    Assert.assertEquals("me", grantInfo.getGrantor());
-    Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
-    Assert.assertFalse(grantInfo.isGrantOption());
-
-    Assert.assertEquals(2, pps.getRolePrivilegesSize());
-    Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[0]).size());
-    grantInfo = pps.getRolePrivileges().get(roleNames[0]).get(0);
-    Assert.assertEquals("write", grantInfo.getPrivilege());
-    Assert.assertTrue(now <= grantInfo.getCreateTime());
-    Assert.assertEquals("me", grantInfo.getGrantor());
-    Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
-    Assert.assertTrue(grantInfo.isGrantOption());
-
-    Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[1]).size());
-    grantInfo = pps.getRolePrivileges().get(roleNames[1]).get(0);
-    Assert.assertEquals("exec", grantInfo.getPrivilege());
-    Assert.assertTrue(now <= grantInfo.getCreateTime());
-    Assert.assertEquals("me", grantInfo.getGrantor());
-    Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
-    Assert.assertFalse(grantInfo.isGrantOption());
-
-    pps = getPPS(objectType, dbName, tableName, userNames[1]);
-
-    Assert.assertEquals(0, pps.getUserPrivilegesSize());
-
-    Assert.assertEquals(1, pps.getRolePrivilegesSize());
-    Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[1]).size());
-    grantInfo = pps.getRolePrivileges().get(roleNames[1]).get(0);
-    Assert.assertEquals("exec", grantInfo.getPrivilege());
-    Assert.assertTrue(now <= grantInfo.getCreateTime());
-    Assert.assertEquals("me", grantInfo.getGrantor());
-    Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
-    Assert.assertFalse(grantInfo.isGrantOption());
-
-    pps = getPPS(objectType, dbName, tableName, userNames[2]);
-
-    Assert.assertEquals(1, pps.getUserPrivilegesSize());
-    Assert.assertEquals(2, pps.getUserPrivileges().get(userNames[2]).size());
-    Assert.assertEquals(0, pps.getRolePrivilegesSize());
-
-    pps = getPPS(objectType, dbName, tableName, userNames[3]);
-    Assert.assertEquals(0, pps.getUserPrivilegesSize());
-    Assert.assertEquals(0, pps.getRolePrivilegesSize());
-
-    // Test that removing role removes the role grants
-    store.removeRole(roleNames[1]);
-    checkRoleRemovedFromAllPrivileges(objectType, dbName, tableName, roleNames[1]);
-    pps = getPPS(objectType, dbName, tableName, userNames[0]);
-
-    Assert.assertEquals(1, pps.getRolePrivilegesSize());
-    Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[0]).size());
-
-    pps = getPPS(objectType, dbName, tableName, userNames[1]);
-
-    Assert.assertEquals(0, pps.getRolePrivilegesSize());
-
-    // Test that revoking with grant option = true just removes grant option
-    privileges.clear();
-    hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
-    grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
-    hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
-    privileges.add(hop);
-
-    hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
-    grantInfo = new PrivilegeGrantInfo("create2", now, "me", PrincipalType.USER, true);
-    hop = new HiveObjectPrivilege(hiveObjRef, userNames[2], PrincipalType.USER, grantInfo);
-    privileges.add(hop);
-
-    pBag = new PrivilegeBag(privileges);
-    store.revokePrivileges(pBag, true);
-    pps = getPPS(objectType, dbName, tableName, userNames[0]);
-
-    Assert.assertEquals(1, pps.getRolePrivilegesSize());
-    Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[0]).size());
-    grantInfo = pps.getRolePrivileges().get(roleNames[0]).get(0);
-    Assert.assertEquals("write", grantInfo.getPrivilege());
-    Assert.assertTrue(now <= grantInfo.getCreateTime());
-    Assert.assertEquals("me", grantInfo.getGrantor());
-    Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
-    Assert.assertFalse(grantInfo.isGrantOption());
-
-    pps = getPPS(objectType, dbName, tableName, userNames[2]);
-
-    Assert.assertEquals(1, pps.getUserPrivilegesSize());
-    Assert.assertEquals(2, pps.getUserPrivileges().get(userNames[2]).size());
-    for (PrivilegeGrantInfo pgi : pps.getUserPrivileges().get(userNames[2])) {
-      if (pgi.getPrivilege().equals("create")) Assert.assertTrue(pgi.isGrantOption());
-      else if (pgi.getPrivilege().equals("create2")) Assert.assertFalse(pgi.isGrantOption());
-      else Assert.fail("huh?");
-    }
-
-    // Test revoking revokes
-    store.revokePrivileges(pBag, false);
-
-    pps = getPPS(objectType, dbName, tableName, userNames[0]);
-
-    Assert.assertEquals(1, pps.getUserPrivilegesSize());
-    Assert.assertEquals(1, pps.getRolePrivilegesSize());
-    Assert.assertEquals(0, pps.getRolePrivileges().get(roleNames[0]).size());
-
-    pps = getPPS(objectType, dbName, tableName, userNames[2]);
-    Assert.assertEquals(1, pps.getUserPrivilegesSize());
-    Assert.assertEquals(1, pps.getUserPrivileges().get(userNames[2]).size());
-    Assert.assertEquals("create", pps.getUserPrivileges().get(userNames[2]).get(0).getPrivilege());
-    Assert.assertEquals(0, pps.getRolePrivilegesSize());
-  }
-
-  private PrincipalPrivilegeSet getPPS(HiveObjectType objectType, String dbName, String tableName,
-                                       String userName)
-      throws InvalidObjectException, MetaException {
-    switch (objectType) {
-      case GLOBAL: return store.getUserPrivilegeSet(userName, null);
-      case DATABASE: return store.getDBPrivilegeSet(dbName, userName, null);
-      case TABLE: return store.getTablePrivilegeSet(dbName, tableName, userName, null);
-      default: throw new RuntimeException("huh?");
-    }
-  }
-
-  private void checkRoleRemovedFromAllPrivileges(HiveObjectType objectType, String dbName,
-                                                 String tableName, String roleName)
-      throws IOException, NoSuchObjectException, MetaException {
-    List<PrivilegeGrantInfo> pgi = null;
-    switch (objectType) {
-      case GLOBAL:
-        pgi = HBaseReadWrite.getInstance().getGlobalPrivs().getRolePrivileges().get(roleName);
-        break;
-
-      case DATABASE:
-        pgi = store.getDatabase(dbName).getPrivileges().getRolePrivileges().get(roleName);
-        break;
-
-      case TABLE:
-        pgi = store.getTable(dbName, tableName).getPrivileges().getRolePrivileges().get(roleName);
-        break;
-
-      default:
-        Assert.fail();
-    }
-
-    Assert.assertNull("Expected null for role " + roleName + " for type " + objectType.toString()
-      + " with db " + dbName + " and table " + tableName, pgi);
-  }
-
-  @Test
-  public void listDbGrants() throws Exception {
-    String dbNames[] = new String[] {"ldbg_db1", "ldbg_db2"};
-    try {
-      Database db = new Database(dbNames[0], "no description", "file:///tmp", emptyParameters);
-      store.createDatabase(db);
-      db = new Database(dbNames[1], "no description", "file:///tmp", emptyParameters);
-      store.createDatabase(db);
-      String[] roleNames = new String[]{"ldbg_role1", "ldbg_role2"};
-      String[] userNames = new String[]{"frodo", "sam"};
-
-      store.addRole(roleNames[0], "me");
-      store.addRole(roleNames[1], "me");
-      int now = (int)(System.currentTimeMillis() / 1000);
-
-      Role role1 = store.getRole(roleNames[0]);
-      Role role2 = store.getRole(roleNames[1]);
-      store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false);
-      store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
-      store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false);
-
-      List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-      HiveObjectRef hiveObjRef =
-          new HiveObjectRef(HiveObjectType.DATABASE, dbNames[0], null, null, null);
-      PrivilegeGrantInfo grantInfo =
-          new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false);
-      HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER,
-          grantInfo);
-      privileges.add(hop);
-
-      grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
-      hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
-      privileges.add(hop);
-
-      PrivilegeBag pBag = new PrivilegeBag(privileges);
-      store.grantPrivileges(pBag);
-
-      List<HiveObjectPrivilege> hops =
-          store.listPrincipalDBGrants(roleNames[0], PrincipalType.ROLE, dbNames[0]);
-      Assert.assertEquals(1, hops.size());
-      Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
-      Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType());
-      Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
-      hops = store.listPrincipalDBGrants(userNames[0], PrincipalType.USER, dbNames[0]);
-      Assert.assertEquals(1, hops.size());
-      Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
-      Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType());
-      Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
-      hops = store.listPrincipalDBGrants(roleNames[1], PrincipalType.ROLE, dbNames[0]);
-      Assert.assertEquals(0, hops.size());
-      hops = store.listPrincipalDBGrants(userNames[1], PrincipalType.USER, dbNames[0]);
-      Assert.assertEquals(0, hops.size());
-
-      hops = store.listPrincipalDBGrants(roleNames[0], PrincipalType.ROLE, dbNames[1]);
-      Assert.assertEquals(0, hops.size());
-      hops = store.listPrincipalDBGrants(userNames[0], PrincipalType.USER, dbNames[1]);
-      Assert.assertEquals(0, hops.size());
-
-      hops = store.listDBGrantsAll(dbNames[0]);
-      Assert.assertEquals(2, hops.size());
-      boolean sawUser = false, sawRole = false;
-      for (HiveObjectPrivilege h : hops) {
-        if (h.getPrincipalName().equals(userNames[0])) {
-          Assert.assertEquals(PrincipalType.USER, h.getPrincipalType());
-          Assert.assertEquals(HiveObjectType.DATABASE, h.getHiveObject().getObjectType());
-          Assert.assertEquals("read", h.getGrantInfo().getPrivilege());
-          sawUser = true;
-        } else if (h.getPrincipalName().equals(roleNames[0])) {
-          Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType());
-          Assert.assertEquals(HiveObjectType.DATABASE, h.getHiveObject().getObjectType());
-          Assert.assertEquals("write", h.getGrantInfo().getPrivilege());
-          sawRole = true;
-        }
-      }
-      Assert.assertTrue(sawUser && sawRole);
-
-      hops = store.listPrincipalDBGrantsAll(roleNames[0], PrincipalType.ROLE);
-      Assert.assertEquals(1, hops.size());
-      Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
-      Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType());
-      Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
-      hops = store.listPrincipalDBGrantsAll(userNames[0], PrincipalType.USER);
-      Assert.assertEquals(1, hops.size());
-      Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
-      Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType());
-      Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
-      hops = store.listPrincipalDBGrantsAll(roleNames[1], PrincipalType.ROLE);
-      Assert.assertEquals(0, hops.size());
-      hops = store.listPrincipalDBGrantsAll(userNames[1], PrincipalType.USER);
-      Assert.assertEquals(0, hops.size());
-
-
-    } finally {
-      store.dropDatabase(dbNames[0]);
-      store.dropDatabase(dbNames[1]);
-    }
-  }
-
-  @Test
-  public void listGlobalGrants() throws Exception {
-    String[] roleNames = new String[]{"lgg_role1", "lgg_role2"};
-    String[] userNames = new String[]{"merry", "pippen"};
-
-    store.addRole(roleNames[0], "me");
-    store.addRole(roleNames[1], "me");
-    int now = (int)(System.currentTimeMillis() / 1000);
-
-    Role role1 = store.getRole(roleNames[0]);
-    Role role2 = store.getRole(roleNames[1]);
-    store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false);
-    store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
-    store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false);
-
-    List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-    HiveObjectRef hiveObjRef =
-        new HiveObjectRef(HiveObjectType.GLOBAL, null, null, null, null);
-    PrivilegeGrantInfo grantInfo =
-        new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false);
-    HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER,
-        grantInfo);
-    privileges.add(hop);
-
-    grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
-    hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
-    privileges.add(hop);
-
-    PrivilegeBag pBag = new PrivilegeBag(privileges);
-    store.grantPrivileges(pBag);
-
-    List<HiveObjectPrivilege> hops =
-        store.listPrincipalGlobalGrants(roleNames[0], PrincipalType.ROLE);
-    Assert.assertEquals(1, hops.size());
-    Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
-    Assert.assertEquals(HiveObjectType.GLOBAL, hops.get(0).getHiveObject().getObjectType());
-    Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
-    hops = store.listPrincipalGlobalGrants(userNames[0], PrincipalType.USER);
-    Assert.assertEquals(1, hops.size());
-    Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
-    Assert.assertEquals(HiveObjectType.GLOBAL, hops.get(0).getHiveObject().getObjectType());
-    Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
-    hops = store.listPrincipalGlobalGrants(roleNames[1], PrincipalType.ROLE);
-    Assert.assertEquals(0, hops.size());
-    hops = store.listPrincipalGlobalGrants(userNames[1], PrincipalType.USER);
-    Assert.assertEquals(0, hops.size());
-
-    hops = store.listGlobalGrantsAll();
-    Assert.assertEquals(2, hops.size());
-    boolean sawUser = false, sawRole = false;
-    for (HiveObjectPrivilege h : hops) {
-      if (h.getPrincipalName().equals(userNames[0])) {
-        Assert.assertEquals(PrincipalType.USER, h.getPrincipalType());
-        Assert.assertEquals(HiveObjectType.GLOBAL, h.getHiveObject().getObjectType());
-        Assert.assertEquals("read", h.getGrantInfo().getPrivilege());
-        sawUser = true;
-      } else if (h.getPrincipalName().equals(roleNames[0])) {
-        Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType());
-        Assert.assertEquals(HiveObjectType.GLOBAL, h.getHiveObject().getObjectType());
-        Assert.assertEquals("write", h.getGrantInfo().getPrivilege());
-        sawRole = true;
-      }
-    }
-    Assert.assertTrue(sawUser && sawRole);
-  }
-
-  @Test
-  public void listTableGrants() throws Exception {
-    String dbName = "ltg_db";
-    String[] tableNames = new String[] {"ltg_t1", "ltg_t2"};
-    try {
-      Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters);
-      store.createDatabase(db);
-      int startTime = (int)(System.currentTimeMillis() / 1000);
-      List<FieldSchema> cols = new ArrayList<FieldSchema>();
-      cols.add(new FieldSchema("col1", "int", "nocomment"));
-      SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-      StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-          serde, null, null, emptyParameters);
-      Table table = new Table(tableNames[0], dbName, "me", startTime, startTime, 0, sd, null,
-          emptyParameters, null, null, null);
-      store.createTable(table);
-      table = new Table(tableNames[1], dbName, "me", startTime, startTime, 0, sd, null,
-          emptyParameters, null, null, null);
-      store.createTable(table);
-      String[] roleNames = new String[]{"ltg_role1", "ltg_role2"};
-      String[] userNames = new String[]{"gandalf", "radagast"};
-
-      store.addRole(roleNames[0], "me");
-      store.addRole(roleNames[1], "me");
-      int now = (int)(System.currentTimeMillis() / 1000);
-
-      Role role1 = store.getRole(roleNames[0]);
-      Role role2 = store.getRole(roleNames[1]);
-      store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false);
-      store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
-      store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false);
-
-      List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
-      HiveObjectRef hiveObjRef =
-          new HiveObjectRef(HiveObjectType.TABLE, dbName, tableNames[0], null, null);
-      PrivilegeGrantInfo grantInfo =
-          new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false);
-      HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER,
-          grantInfo);
-      privileges.add(hop);
-
-      grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
-      hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
-      privileges.add(hop);
-
-      PrivilegeBag pBag = new PrivilegeBag(privileges);
-      store.grantPrivileges(pBag);
-
-      List<HiveObjectPrivilege> hops =
-          store.listAllTableGrants(roleNames[0], PrincipalType.ROLE, dbName, tableNames[0]);
-      Assert.assertEquals(1, hops.size());
-      Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
-      Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
-      Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
-      hops = store.listAllTableGrants(userNames[0], PrincipalType.USER, dbName, tableNames[0]);
-      Assert.assertEquals(1, hops.size());
-      Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
-      Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
-      Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
-      hops = store.listAllTableGrants(roleNames[1], PrincipalType.ROLE, dbName, tableNames[0]);
-      Assert.assertEquals(0, hops.size());
-      hops = store.listAllTableGrants(userNames[1], PrincipalType.USER, dbName, tableNames[0]);
-      Assert.assertEquals(0, hops.size());
-
-      hops = store.listAllTableGrants(roleNames[0], PrincipalType.ROLE, dbName, tableNames[1]);
-      Assert.assertEquals(0, hops.size());
-      hops = store.listAllTableGrants(userNames[0], PrincipalType.USER, dbName, tableNames[1]);
-      Assert.assertEquals(0, hops.size());
-
-      hops = store.listTableGrantsAll(dbName, tableNames[0]);
-      Assert.assertEquals(2, hops.size());
-      boolean sawUser = false, sawRole = false;
-      for (HiveObjectPrivilege h : hops) {
-        if (h.getPrincipalName().equals(userNames[0])) {
-          Assert.assertEquals(PrincipalType.USER, h.getPrincipalType());
-          Assert.assertEquals(HiveObjectType.TABLE, h.getHiveObject().getObjectType());
-          Assert.assertEquals("read", h.getGrantInfo().getPrivilege());
-          sawUser = true;
-        } else if (h.getPrincipalName().equals(roleNames[0])) {
-          Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType());
-          Assert.assertEquals(HiveObjectType.TABLE, h.getHiveObject().getObjectType());
-          Assert.assertEquals("write", h.getGrantInfo().getPrivilege());
-          sawRole = true;
-        }
-      }
-      Assert.assertTrue(sawUser && sawRole);
-
-      hops = store.listPrincipalTableGrantsAll(roleNames[0], PrincipalType.ROLE);
-      Assert.assertEquals(1, hops.size());
-      Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
-      Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
-      Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
-      hops = store.listPrincipalTableGrantsAll(userNames[0], PrincipalType.USER);
-      Assert.assertEquals(1, hops.size());
-      Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
-      Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
-      Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
-      hops = store.listPrincipalDBGrantsAll(roleNames[1], PrincipalType.ROLE);
-      Assert.assertEquals(0, hops.size());
-      hops = store.listPrincipalDBGrantsAll(userNames[1], PrincipalType.USER);
-      Assert.assertEquals(0, hops.size());
-
-
-    } finally {
-      store.dropTable(dbName, tableNames[0]);
-      store.dropTable(dbName, tableNames[1]);
-      store.dropDatabase(dbName);
-    }
-  }
-
-  @Test
-  public void tableStatistics() throws Exception {
-    long now = System.currentTimeMillis();
-    String dbname = "default";
-    String tableName = "statstable";
-    String boolcol = "boolcol";
-    String longcol = "longcol";
-    String doublecol = "doublecol";
-    String stringcol = "stringcol";
-    String binarycol = "bincol";
-    String decimalcol = "deccol";
-    long trues = 37;
-    long falses = 12;
-    long booleanNulls = 2;
-    long longHigh = 120938479124L;
-    long longLow = -12341243213412124L;
-    long longNulls = 23;
-    long longDVs = 213L;
-    double doubleHigh = 123423.23423;
-    double doubleLow = 0.00001234233;
-    long doubleNulls = 92;
-    long doubleDVs = 1234123421L;
-    long strMaxLen = 1234;
-    double strAvgLen = 32.3;
-    long strNulls = 987;
-    long strDVs = 906;
-    long binMaxLen = 123412987L;
-    double binAvgLen = 76.98;
-    long binNulls = 976998797L;
-    Decimal decHigh = new Decimal();
-    decHigh.setScale((short)3);
-    decHigh.setUnscaled("3876".getBytes()); // I have no clue how this is translated, but it
-    // doesn't matter
-    Decimal decLow = new Decimal();
-    decLow.setScale((short)3);
-    decLow.setUnscaled("38".getBytes());
-    long decNulls = 13;
-    long decDVs = 923947293L;
-
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema(boolcol, "boolean", "nocomment"));
-    cols.add(new FieldSchema(longcol, "long", "nocomment"));
-    cols.add(new FieldSchema(doublecol, "double", "nocomment"));
-    cols.add(new FieldSchema(stringcol, "varchar(32)", "nocomment"));
-    cols.add(new FieldSchema(binarycol, "binary", "nocomment"));
-    cols.add(new FieldSchema(decimalcol, "decimal(5, 3)", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table(tableName, dbname, "me", (int)now / 1000, (int)now / 1000, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    ColumnStatistics stats = new ColumnStatistics();
-    ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
-    desc.setLastAnalyzed(now);
-    desc.setDbName(dbname);
-    desc.setTableName(tableName);
-    desc.setIsTblLevel(true);
-    stats.setStatsDesc(desc);
-
-    // Do one column of each type
-    ColumnStatisticsObj obj = new ColumnStatisticsObj();
-    obj.setColName(boolcol);
-    obj.setColType("boolean");
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    BooleanColumnStatsData boolData = new BooleanColumnStatsData();
-    boolData.setNumTrues(trues);
-    boolData.setNumFalses(falses);
-    boolData.setNumNulls(booleanNulls);
-    data.setBooleanStats(boolData);
-    obj.setStatsData(data);
-    stats.addToStatsObj(obj);
-
-    obj = new ColumnStatisticsObj();
-    obj.setColName(longcol);
-    obj.setColType("long");
-    data = new ColumnStatisticsData();
-    LongColumnStatsData longData = new LongColumnStatsData();
-    longData.setHighValue(longHigh);
-    longData.setLowValue(longLow);
-    longData.setNumNulls(longNulls);
-    longData.setNumDVs(longDVs);
-    data.setLongStats(longData);
-    obj.setStatsData(data);
-    stats.addToStatsObj(obj);
-
-    obj = new ColumnStatisticsObj();
-    obj.setColName(doublecol);
-    obj.setColType("double");
-    data = new ColumnStatisticsData();
-    DoubleColumnStatsData doubleData = new DoubleColumnStatsData();
-    doubleData.setHighValue(doubleHigh);
-    doubleData.setLowValue(doubleLow);
-    doubleData.setNumNulls(doubleNulls);
-    doubleData.setNumDVs(doubleDVs);
-    data.setDoubleStats(doubleData);
-    obj.setStatsData(data);
-    stats.addToStatsObj(obj);
-
-    store.updateTableColumnStatistics(stats);
-
-    stats = store.getTableColumnStatistics(dbname, tableName,
-        Arrays.asList(boolcol, longcol, doublecol));
-
-    // We'll check all of the individual values later.
-    Assert.assertEquals(3, stats.getStatsObjSize());
-
-    // check that we can fetch just some of the columns
-    stats = store.getTableColumnStatistics(dbname, tableName, Arrays.asList(boolcol));
-    Assert.assertEquals(1, stats.getStatsObjSize());
-
-    stats = new ColumnStatistics();
-    stats.setStatsDesc(desc);
-
-
-    obj = new ColumnStatisticsObj();
-    obj.setColName(stringcol);
-    obj.setColType("string");
-    data = new ColumnStatisticsData();
-    StringColumnStatsData strData = new StringColumnStatsData();
-    strData.setMaxColLen(strMaxLen);
-    strData.setAvgColLen(strAvgLen);
-    strData.setNumNulls(strNulls);
-    strData.setNumDVs(strDVs);
-    data.setStringStats(strData);
-    obj.setStatsData(data);
-    stats.addToStatsObj(obj);
-
-    obj = new ColumnStatisticsObj();
-    obj.setColName(binarycol);
-    obj.setColType("binary");
-    data = new ColumnStatisticsData();
-    BinaryColumnStatsData binData = new BinaryColumnStatsData();
-    binData.setMaxColLen(binMaxLen);
-    binData.setAvgColLen(binAvgLen);
-    binData.setNumNulls(binNulls);
-    data.setBinaryStats(binData);
-    obj.setStatsData(data);
-    stats.addToStatsObj(obj);
-
-    obj = new ColumnStatisticsObj();
-    obj.setColName(decimalcol);
-    obj.setColType("decimal(5,3)");
-    data = new ColumnStatisticsData();
-    DecimalColumnStatsData decData = new DecimalColumnStatsData();
-    LOG.debug("Setting decimal high value to " + decHigh.getScale() + " <" + new String(decHigh.getUnscaled()) + ">");
-    decData.setHighValue(decHigh);
-    decData.setLowValue(decLow);
-    decData.setNumNulls(decNulls);
-    decData.setNumDVs(decDVs);
-    data.setDecimalStats(decData);
-    obj.setStatsData(data);
-    stats.addToStatsObj(obj);
-
-    store.updateTableColumnStatistics(stats);
-
-    stats = store.getTableColumnStatistics(dbname, tableName,
-        Arrays.asList(boolcol, longcol, doublecol, stringcol, binarycol, decimalcol));
-    Assert.assertEquals(now, stats.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(dbname, stats.getStatsDesc().getDbName());
-    Assert.assertEquals(tableName, stats.getStatsDesc().getTableName());
-    Assert.assertTrue(stats.getStatsDesc().isIsTblLevel());
-
-    Assert.assertEquals(6, stats.getStatsObjSize());
-
-    ColumnStatisticsData colData = stats.getStatsObj().get(0).getStatsData();
-    Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS, colData.getSetField());
-    boolData = colData.getBooleanStats();
-    Assert.assertEquals(trues, boolData.getNumTrues());
-    Assert.assertEquals(falses, boolData.getNumFalses());
-    Assert.assertEquals(booleanNulls, boolData.getNumNulls());
-
-    colData = stats.getStatsObj().get(1).getStatsData();
-    Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, colData.getSetField());
-    longData = colData.getLongStats();
-    Assert.assertEquals(longHigh, longData.getHighValue());
-    Assert.assertEquals(longLow, longData.getLowValue());
-    Assert.assertEquals(longNulls, longData.getNumNulls());
-    Assert.assertEquals(longDVs, longData.getNumDVs());
-
-    colData = stats.getStatsObj().get(2).getStatsData();
-    Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, colData.getSetField());
-    doubleData = colData.getDoubleStats();
-    Assert.assertEquals(doubleHigh, doubleData.getHighValue(), 0.01);
-    Assert.assertEquals(doubleLow, doubleData.getLowValue(), 0.01);
-    Assert.assertEquals(doubleNulls, doubleData.getNumNulls());
-    Assert.assertEquals(doubleDVs, doubleData.getNumDVs());
-
-    colData = stats.getStatsObj().get(3).getStatsData();
-    Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, colData.getSetField());
-    strData = colData.getStringStats();
-    Assert.assertEquals(strMaxLen, strData.getMaxColLen());
-    Assert.assertEquals(strAvgLen, strData.getAvgColLen(), 0.01);
-    Assert.assertEquals(strNulls, strData.getNumNulls());
-    Assert.assertEquals(strDVs, strData.getNumDVs());
-
-    colData = stats.getStatsObj().get(4).getStatsData();
-    Assert.assertEquals(ColumnStatisticsData._Fields.BINARY_STATS, colData.getSetField());
-    binData = colData.getBinaryStats();
-    Assert.assertEquals(binMaxLen, binData.getMaxColLen());
-    Assert.assertEquals(binAvgLen, binData.getAvgColLen(), 0.01);
-    Assert.assertEquals(binNulls, binData.getNumNulls());
-
-    colData = stats.getStatsObj().get(5).getStatsData();
-    Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, colData.getSetField());
-    decData = colData.getDecimalStats();
-    Assert.assertEquals(decHigh, decData.getHighValue());
-    Assert.assertEquals(decLow, decData.getLowValue());
-    Assert.assertEquals(decNulls, decData.getNumNulls());
-    Assert.assertEquals(decDVs, decData.getNumDVs());
-
-  }
-
-  @Test
-  public void partitionStatistics() throws Exception {
-    long now = System.currentTimeMillis();
-    String dbname = "default";
-    String tableName = "statspart";
-    String[] partNames = {"ds=today", "ds=yesterday"};
-    String[] partVals = {"today", "yesterday"};
-    String boolcol = "boolcol";
-    String longcol = "longcol";
-    String doublecol = "doublecol";
-    String stringcol = "stringcol";
-    String binarycol = "bincol";
-    String decimalcol = "deccol";
-    long trues = 37;
-    long falses = 12;
-    long booleanNulls = 2;
-    long strMaxLen = 1234;
-    double strAvgLen = 32.3;
-    long strNulls = 987;
-    long strDVs = 906;
-
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema(boolcol, "boolean", "nocomment"));
-    cols.add(new FieldSchema(longcol, "long", "nocomment"));
-    cols.add(new FieldSchema(doublecol, "double", "nocomment"));
-    cols.add(new FieldSchema(stringcol, "varchar(32)", "nocomment"));
-    cols.add(new FieldSchema(binarycol, "binary", "nocomment"));
-    cols.add(new FieldSchema(decimalcol, "decimal(5, 3)", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbname, "me", (int)now / 1000, (int)now / 1000, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-    for (String partVal : partVals) {
-      Partition part = new Partition(Arrays.asList(partVal), dbname, tableName, (int) now / 1000,
-          (int) now / 1000, sd, emptyParameters);
-      store.addPartition(part);
-    }
-    
-    for (int i = 0; i < partNames.length; i++) {
-      ColumnStatistics stats = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
-      desc.setLastAnalyzed(now);
-      desc.setDbName(dbname);
-      desc.setTableName(tableName);
-      desc.setIsTblLevel(false);
-      desc.setPartName(partNames[i]);
-      stats.setStatsDesc(desc);
-
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName(boolcol);
-      obj.setColType("boolean");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      BooleanColumnStatsData boolData = new BooleanColumnStatsData();
-      boolData.setNumTrues(trues);
-      boolData.setNumFalses(falses);
-      boolData.setNumNulls(booleanNulls);
-      data.setBooleanStats(boolData);
-      obj.setStatsData(data);
-      stats.addToStatsObj(obj);
-
-      store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i]));
-    }
-
-    List<ColumnStatistics> statsList = store.getPartitionColumnStatistics(dbname, tableName,
-        Arrays.asList(partNames), Arrays.asList(boolcol));
-
-    Assert.assertEquals(2, statsList.size());
-    for (int i = 0; i < partNames.length; i++) {
-      Assert.assertEquals(1, statsList.get(i).getStatsObjSize());
-    }
-
-    for (int i = 0; i < partNames.length; i++) {
-      ColumnStatistics stats = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
-      desc.setLastAnalyzed(now);
-      desc.setDbName(dbname);
-      desc.setTableName(tableName);
-      desc.setIsTblLevel(false);
-      desc.setPartName(partNames[i]);
-      stats.setStatsDesc(desc);
-
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName(stringcol);
-      obj.setColType("string");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      StringColumnStatsData strData = new StringColumnStatsData();
-      strData.setMaxColLen(strMaxLen);
-      strData.setAvgColLen(strAvgLen);
-      strData.setNumNulls(strNulls);
-      strData.setNumDVs(strDVs);
-      data.setStringStats(strData);
-      obj.setStatsData(data);
-      stats.addToStatsObj(obj);
-
-      store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i]));
-    }
-
-    // Make sure when we ask for one we only get one
-    statsList = store.getPartitionColumnStatistics(dbname, tableName,
-        Arrays.asList(partNames), Arrays.asList(boolcol));
-
-    Assert.assertEquals(2, statsList.size());
-    for (int i = 0; i < partNames.length; i++) {
-      Assert.assertEquals(1, statsList.get(i).getStatsObjSize());
-    }
-
-    statsList = store.getPartitionColumnStatistics(dbname, tableName,
-        Arrays.asList(partNames), Arrays.asList(boolcol, stringcol));
-
-    Assert.assertEquals(2, statsList.size());
-    for (int i = 0; i < partNames.length; i++) {
-      Assert.assertEquals(2, statsList.get(i).getStatsObjSize());
-      // Just check one piece of the data, I don't need to check it all again
-      Assert.assertEquals(booleanNulls,
-          statsList.get(i).getStatsObj().get(0).getStatsData().getBooleanStats().getNumNulls());
-      Assert.assertEquals(strDVs,
-          statsList.get(i).getStatsObj().get(1).getStatsData().getStringStats().getNumDVs());
-    }
-  }
-
-  @Test
-  public void delegationToken() throws Exception {
-    store.addToken("abc", "def");
-    store.addToken("ghi", "jkl");
-
-    Assert.assertEquals("def", store.getToken("abc"));
-    Assert.assertEquals("jkl", store.getToken("ghi"));
-    Assert.assertNull(store.getToken("wabawaba"));
-    String[] allToks = store.getAllTokenIdentifiers().toArray(new String[2]);
-    Arrays.sort(allToks);
-    Assert.assertArrayEquals(new String[]{"abc", "ghi"}, allToks);
-
-    store.removeToken("abc");
-    store.removeToken("wabawaba");
-
-    Assert.assertNull(store.getToken("abc"));
-    Assert.assertEquals("jkl", store.getToken("ghi"));
-    allToks = store.getAllTokenIdentifiers().toArray(new String[1]);
-    Assert.assertArrayEquals(new String[]{"ghi"}, allToks);
-  }
-
-  @Test
-  public void masterKey() throws Exception {
-    Assert.assertEquals(0, store.addMasterKey("k1"));
-    Assert.assertEquals(1, store.addMasterKey("k2"));
-
-    String[] keys = store.getMasterKeys();
-    Arrays.sort(keys);
-    Assert.assertArrayEquals(new String[]{"k1", "k2"}, keys);
-
-    store.updateMasterKey(0, "k3");
-    keys = store.getMasterKeys();
-    Arrays.sort(keys);
-    Assert.assertArrayEquals(new String[]{"k2", "k3"}, keys);
-
-    store.removeMasterKey(1);
-    keys = store.getMasterKeys();
-    Assert.assertArrayEquals(new String[]{"k3"}, keys);
-
-    thrown.expect(NoSuchObjectException.class);
-    store.updateMasterKey(72, "whatever");
-  }
-
-}


[11/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
new file mode 100644
index 0000000..2542a00
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.Date;
+import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
+
+public class DateColumnStatsMerger extends ColumnStatsMerger {
+  @Override
+  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+    DateColumnStatsData aggregateData = aggregateColStats.getStatsData().getDateStats();
+    DateColumnStatsData newData = newColStats.getStatsData().getDateStats();
+    Date lowValue = aggregateData.getLowValue().compareTo(newData.getLowValue()) < 0 ? aggregateData
+        .getLowValue() : newData.getLowValue();
+    aggregateData.setLowValue(lowValue);
+    Date highValue = aggregateData.getHighValue().compareTo(newData.getHighValue()) >= 0 ? aggregateData
+        .getHighValue() : newData.getHighValue();
+    aggregateData.setHighValue(highValue);
+    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+    } else {
+      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(newData.getBitVectors());
+      long ndv = -1;
+      if (oldEst.canMerge(newEst)) {
+        oldEst.mergeEstimators(newEst);
+        ndv = oldEst.estimateNumDistinctValues();
+        aggregateData.setBitVectors(oldEst.serialize());
+      } else {
+        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+      }
+      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+      aggregateData.setNumDVs(ndv);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
new file mode 100644
index 0000000..4e8e129
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.Decimal;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+
+public class DecimalColumnStatsMerger extends ColumnStatsMerger {
+  @Override
+  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+    DecimalColumnStatsData aggregateData = aggregateColStats.getStatsData().getDecimalStats();
+    DecimalColumnStatsData newData = newColStats.getStatsData().getDecimalStats();
+    Decimal lowValue = aggregateData.getLowValue() != null
+        && (aggregateData.getLowValue().compareTo(newData.getLowValue()) > 0) ? aggregateData
+        .getLowValue() : newData.getLowValue();
+    aggregateData.setLowValue(lowValue);
+    Decimal highValue = aggregateData.getHighValue() != null
+        && (aggregateData.getHighValue().compareTo(newData.getHighValue()) > 0) ? aggregateData
+        .getHighValue() : newData.getHighValue();
+    aggregateData.setHighValue(highValue);
+    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+    } else {
+      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(newData.getBitVectors());
+      long ndv = -1;
+      if (oldEst.canMerge(newEst)) {
+        oldEst.mergeEstimators(newEst);
+        ndv = oldEst.estimateNumDistinctValues();
+        aggregateData.setBitVectors(oldEst.serialize());
+      } else {
+        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+      }
+      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+      aggregateData.setNumDVs(ndv);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
new file mode 100644
index 0000000..4ef5c39
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+
+public class DoubleColumnStatsMerger extends ColumnStatsMerger {
+  @Override
+  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+    DoubleColumnStatsData aggregateData = aggregateColStats.getStatsData().getDoubleStats();
+    DoubleColumnStatsData newData = newColStats.getStatsData().getDoubleStats();
+    aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+    aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+    } else {
+      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(newData.getBitVectors());
+      long ndv = -1;
+      if (oldEst.canMerge(newEst)) {
+        oldEst.mergeEstimators(newEst);
+        ndv = oldEst.estimateNumDistinctValues();
+        aggregateData.setBitVectors(oldEst.serialize());
+      } else {
+        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+      }
+      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+      aggregateData.setNumDVs(ndv);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
new file mode 100644
index 0000000..acf7f03
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+
+public class LongColumnStatsMerger extends ColumnStatsMerger {
+  @Override
+  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+    LongColumnStatsData aggregateData = aggregateColStats.getStatsData().getLongStats();
+    LongColumnStatsData newData = newColStats.getStatsData().getLongStats();
+    aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+    aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+    } else {
+      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(newData.getBitVectors());
+      long ndv = -1;
+      if (oldEst.canMerge(newEst)) {
+        oldEst.mergeEstimators(newEst);
+        ndv = oldEst.estimateNumDistinctValues();
+        aggregateData.setBitVectors(oldEst.serialize());
+      } else {
+        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+      }
+      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+      aggregateData.setNumDVs(ndv);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
new file mode 100644
index 0000000..b3cd33c
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+
+public class StringColumnStatsMerger extends ColumnStatsMerger {
+  @Override
+  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+    StringColumnStatsData aggregateData = aggregateColStats.getStatsData().getStringStats();
+    StringColumnStatsData newData = newColStats.getStatsData().getStringStats();
+    aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
+    aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
+    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+    } else {
+      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+          .getNumDistinctValueEstimator(newData.getBitVectors());
+      long ndv = -1;
+      if (oldEst.canMerge(newEst)) {
+        oldEst.mergeEstimators(newEst);
+        ndv = oldEst.estimateNumDistinctValues();
+        aggregateData.setBitVectors(oldEst.serialize());
+      } else {
+        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+      }
+      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+      aggregateData.setNumDVs(ndv);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/AggrStatsInvalidatorFilter.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/AggrStatsInvalidatorFilter.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/AggrStatsInvalidatorFilter.java
deleted file mode 100644
index 2db5c38..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/AggrStatsInvalidatorFilter.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.primitives.Longs;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hive.common.util.BloomFilter;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Filter for scanning aggregates stats table
- */
-public class AggrStatsInvalidatorFilter extends FilterBase {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(AggrStatsInvalidatorFilter.class.getName());
-  private final List<HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry> entries;
-  private final long runEvery;
-  private final long maxCacheEntryLife;
-  // This class is not serializable, so I realize transient doesn't mean anything.  It's just to
-  // comunicate that we don't serialize this and ship it across to the filter on the other end.
-  // We use the time the filter is actually instantiated in HBase.
-  private transient long now;
-
-  public static Filter parseFrom(byte[] serialized) throws DeserializationException {
-    try {
-      return new AggrStatsInvalidatorFilter(
-          HbaseMetastoreProto.AggrStatsInvalidatorFilter.parseFrom(serialized));
-    } catch (InvalidProtocolBufferException e) {
-      throw new DeserializationException(e);
-    }
-  }
-
-  /**
-   * @param proto Protocol buffer representation of this filter.
-   */
-  AggrStatsInvalidatorFilter(HbaseMetastoreProto.AggrStatsInvalidatorFilter proto) {
-    this.entries = proto.getToInvalidateList();
-    this.runEvery = proto.getRunEvery();
-    this.maxCacheEntryLife = proto.getMaxCacheEntryLife();
-    now = System.currentTimeMillis();
-  }
-
-  @Override
-  public byte[] toByteArray() throws IOException {
-    return HbaseMetastoreProto.AggrStatsInvalidatorFilter.newBuilder()
-        .addAllToInvalidate(entries)
-        .setRunEvery(runEvery)
-        .setMaxCacheEntryLife(maxCacheEntryLife)
-        .build()
-        .toByteArray();
-  }
-
-  @Override
-  public boolean filterAllRemaining() throws IOException {
-    return false;
-  }
-
-  @Override
-  public ReturnCode filterKeyValue(Cell cell) throws IOException {
-    // Is this the partition we want?
-    if (Arrays.equals(CellUtil.cloneQualifier(cell), HBaseReadWrite.AGGR_STATS_BLOOM_COL)) {
-      HbaseMetastoreProto.AggrStatsBloomFilter fromCol =
-          HbaseMetastoreProto.AggrStatsBloomFilter.parseFrom(CellUtil.cloneValue(cell));
-      BloomFilter bloom = null;
-      if (now - maxCacheEntryLife > fromCol.getAggregatedAt()) {
-        // It's too old, kill it regardless of whether we were asked to or not.
-        return ReturnCode.INCLUDE;
-      } else if (now - runEvery * 2 <= fromCol.getAggregatedAt()) {
-        // It's too new.  We might be stomping on something that was just created.  Skip it.
-        return ReturnCode.NEXT_ROW;
-      } else {
-        // Look through each of our entries and see if any of them match.
-        for (HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry entry : entries) {
-          // First check if we match on db and table match
-          if (entry.getDbName().equals(fromCol.getDbName()) &&
-              entry.getTableName().equals(fromCol.getTableName())) {
-            if (bloom == null) {
-              // Now, reconstitute the bloom filter and probe it with each of our partition names
-              List<Long> bitsList = fromCol.getBloomFilter().getBitsList();
-              bloom = new BloomFilter(Longs.toArray(bitsList), fromCol.getBloomFilter().getNumFuncs());
-            }
-            if (bloom.test(entry.getPartName().toByteArray())) {
-              // This is most likely a match, so mark it and quit looking.
-              return ReturnCode.INCLUDE;
-            }
-          }
-        }
-      }
-      return ReturnCode.NEXT_ROW;
-    } else {
-      return ReturnCode.NEXT_COL;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/Counter.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/Counter.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/Counter.java
deleted file mode 100644
index 2359939..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/Counter.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * A simple metric to count how many times something occurs.
- */
-class Counter {
-  private final String name;
-  private long cnt;
-
-  Counter(String name) {
-    this.name = name;
-    cnt = 0;
-  }
-
-  void incr() {
-    cnt++;
-  }
-
-  void clear() {
-    cnt = 0;
-  }
-
-  String dump() {
-    StringBuilder bldr = new StringBuilder("Dumping metric: ");
-    bldr.append(name).append(' ').append(cnt);
-    return bldr.toString();
-  }
-
-  @VisibleForTesting long getCnt() {
-    return cnt;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseConnection.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseConnection.java
deleted file mode 100644
index 696e588..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseConnection.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * A connection to HBase.  Separated out as an interface so we can slide different transaction
- * managers between our code and HBase.
- */
-public interface HBaseConnection extends Configurable {
-
-  /**
-   * Connects to HBase.  This must be called after {@link #setConf} has been called.
-   * @throws IOException
-   */
-  void connect() throws IOException;
-
-  /**
-   * Close the connection.  No further operations are possible after this is done.
-   * @throws IOException
-   */
-  void close() throws IOException;
-
-  /**
-   * Begin a transaction.
-   * @throws IOException
-   */
-  void beginTransaction() throws IOException;
-
-  /**
-   * Commit a transaction
-   * @throws IOException indicates the commit has failed
-   */
-  void commitTransaction() throws IOException;
-
-  /**
-   * Rollback a transaction
-   * @throws IOException
-   */
-  void rollbackTransaction() throws IOException;
-
-  /**
-   * Flush commits.  A no-op for transaction implementations since they will write at commit time.
-   * @param htab Table to flush
-   * @throws IOException
-   */
-  void flush(HTableInterface htab) throws IOException;
-
-  /**
-   * Create a new table
-   * @param tableName name of the table
-   * @param columnFamilies name of the column families in the table
-   * @throws IOException
-   */
-  void createHBaseTable(String tableName, List<byte[]> columnFamilies) throws IOException;
-
-  /**
-   * Fetch an existing HBase table.
-   * @param tableName name of the table
-   * @return table handle
-   * @throws IOException
-   */
-  HTableInterface getHBaseTable(String tableName) throws IOException;
-
-  /**
-   * Fetch an existing HBase table and force a connection to it.  This should be used only in
-   * cases where you want to assure that the table exists (ie at install).
-   * @param tableName name of the table
-   * @param force if true, force a connection by fetching a non-existant key
-   * @return table handle
-   * @throws IOException
-   */
-  HTableInterface getHBaseTable(String tableName, boolean force) throws IOException;
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseFilterPlanUtil.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseFilterPlanUtil.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseFilterPlanUtil.java
deleted file mode 100644
index 3c03846..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseFilterPlanUtil.java
+++ /dev/null
@@ -1,612 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.hbase.PartitionKeyComparator.Operator;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-
-
-/**
- * Utility function for generating hbase partition filtering plan representation
- * from ExpressionTree.
- * Optimizations to be done -
- *  - Case where all partition keys are specified. Should use a get
- *
- * {@link PartitionFilterGenerator} is a visitor on the given filter expression tree. After
- * walking it it produces the HBase execution plan represented by {@link FilterPlan}. See
- * their javadocs for more details.
- */
-class HBaseFilterPlanUtil {
-
-  /**
-   * Compare two byte arrays.
-   *
-   * @param ar1
-   *          first byte array
-   * @param ar2
-   *          second byte array
-   * @return -1 if ar1 < ar2, 0 if == , 1 if >
-   */
-  static int compare(byte[] ar1, byte[] ar2) {
-    // null check is not needed, nulls are not passed here
-    for (int i = 0; i < ar1.length; i++) {
-      if (i == ar2.length) {
-        return 1;
-      } else {
-        if (ar1[i] == ar2[i]) {
-          continue;
-        } else if (ar1[i] > ar2[i]) {
-          return 1;
-        } else {
-          return -1;
-        }
-      }
-    }
-    // ar2 equal until length of ar1.
-    if(ar1.length == ar2.length) {
-      return 0;
-    }
-    // ar2 has more bytes
-    return -1;
-  }
-
-  /**
-   * Represents the execution plan for hbase to find the set of partitions that
-   * match given filter expression.
-   * If you have an AND or OR of two expressions, you can determine FilterPlan for each
-   * children and then call lhs.and(rhs) or lhs.or(rhs) respectively
-   * to generate a new plan for the expression.
-   *
-   * The execution plan has one or more ScanPlan objects. To get the results the set union of all
-   * ScanPlan objects needs to be done.
-   */
-  public static abstract class FilterPlan {
-    abstract FilterPlan and(FilterPlan other);
-    abstract FilterPlan or(FilterPlan other);
-    abstract List<ScanPlan> getPlans();
-    @Override
-    public String toString() {
-      return getPlans().toString();
-    }
-
-  }
-
-  /**
-   * Represents a union/OR of single scan plans (ScanPlan).
-   */
-  public static class MultiScanPlan extends FilterPlan {
-    final ImmutableList<ScanPlan> scanPlans;
-
-    public MultiScanPlan(List<ScanPlan> scanPlans){
-      this.scanPlans = ImmutableList.copyOf(scanPlans);
-    }
-
-    @Override
-    public FilterPlan and(FilterPlan other) {
-      // Convert to disjunctive normal form (DNF), ie OR of ANDs
-      // First get a new set of FilterPlans by doing an AND
-      // on each ScanPlan in this one with the other FilterPlan
-      List<FilterPlan> newFPlans = new ArrayList<FilterPlan>();
-      for (ScanPlan splan : getPlans()) {
-        newFPlans.add(splan.and(other));
-      }
-      //now combine scanPlans in multiple new FilterPlans into one
-      // MultiScanPlan
-      List<ScanPlan> newScanPlans = new ArrayList<ScanPlan>();
-      for (FilterPlan fp : newFPlans) {
-        newScanPlans.addAll(fp.getPlans());
-      }
-      return new MultiScanPlan(newScanPlans);
-    }
-
-    @Override
-    public FilterPlan or(FilterPlan other) {
-      // just combine the ScanPlans
-      List<ScanPlan> newScanPlans = new ArrayList<ScanPlan>(this.getPlans());
-      newScanPlans.addAll(other.getPlans());
-      return new MultiScanPlan(newScanPlans);
-    }
-
-    @Override
-    public List<ScanPlan> getPlans() {
-      return scanPlans;
-    }
-  }
-
-  /**
-   * Represents a single Hbase Scan api call
-   */
-  public static class ScanPlan extends FilterPlan {
-
-    public static class ScanMarker {
-      final String value;
-      /**
-       * If inclusive = true, it means that the
-       * marker includes those bytes.
-       * If it is false, it means the marker starts at the next possible byte array
-       * or ends at the next possible byte array
-       */
-      final boolean isInclusive;
-      final String type;
-      ScanMarker(String obj, boolean i, String type){
-        this.value = obj;
-        this.isInclusive = i;
-        this.type = type;
-      }
-      @Override
-      public String toString() {
-        return "ScanMarker [" + "value=" + value.toString() + ", isInclusive=" + isInclusive +
-            ", type=" + type + "]";
-      }
-      @Override
-      public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result + value.hashCode();
-        result = prime * result + (isInclusive ? 1231 : 1237);
-        result = prime * result + type.hashCode();
-        return result;
-      }
-      @Override
-      public boolean equals(Object obj) {
-        if (this == obj)
-          return true;
-        if (obj == null)
-          return false;
-        if (getClass() != obj.getClass())
-          return false;
-        ScanMarker other = (ScanMarker) obj;
-        if (!value.equals(other.value))
-          return false;
-        if (isInclusive != other.isInclusive)
-          return false;
-        if (type != other.type)
-          return false;
-        return true;
-      }
-    }
-    public static class ScanMarkerPair {
-      public ScanMarkerPair(ScanMarker startMarker, ScanMarker endMarker) {
-        this.startMarker = startMarker;
-        this.endMarker = endMarker;
-      }
-      ScanMarker startMarker;
-      ScanMarker endMarker;
-    }
-    // represent Scan start, partition key name -> scanMarkerPair
-    Map<String, ScanMarkerPair> markers = new HashMap<String, ScanMarkerPair>();
-    List<Operator> ops = new ArrayList<Operator>();
-
-    // Get the number of partition key prefixes which can be used in the scan range.
-    // For example, if partition key is (year, month, state)
-    // 1. year = 2015 and month >= 1 and month < 5
-    //    year + month can be used in scan range, majorParts = 2
-    // 2. year = 2015 and state = 'CA'
-    //    only year can be used in scan range, majorParts = 1
-    // 3. month = 10 and state = 'CA'
-    //    nothing can be used in scan range, majorParts = 0
-    private int getMajorPartsCount(List<FieldSchema> parts) {
-      int majorPartsCount = 0;
-      while (majorPartsCount<parts.size() && markers.containsKey(parts.get(majorPartsCount).getName())) {
-        ScanMarkerPair pair = markers.get(parts.get(majorPartsCount).getName());
-        majorPartsCount++;
-        if (pair.startMarker!=null && pair.endMarker!=null && pair.startMarker.value.equals(pair
-            .endMarker.value) && pair.startMarker.isInclusive && pair.endMarker.isInclusive) {
-          // is equal
-          continue;
-        } else {
-          break;
-        }
-      }
-      return majorPartsCount;
-    }
-    public Filter getFilter(List<FieldSchema> parts) {
-      int majorPartsCount = getMajorPartsCount(parts);
-      Set<String> majorKeys = new HashSet<String>();
-      for (int i=0;i<majorPartsCount;i++) {
-        majorKeys.add(parts.get(i).getName());
-      }
-
-      List<String> names = HBaseUtils.getPartitionNames(parts);
-      List<PartitionKeyComparator.Range> ranges = new ArrayList<PartitionKeyComparator.Range>();
-      for (Map.Entry<String, ScanMarkerPair> entry : markers.entrySet()) {
-        if (names.contains(entry.getKey()) && !majorKeys.contains(entry.getKey())) {
-          PartitionKeyComparator.Mark startMark = null;
-          if (entry.getValue().startMarker != null) {
-            startMark = new PartitionKeyComparator.Mark(entry.getValue().startMarker.value,
-                entry.getValue().startMarker.isInclusive);
-          }
-          PartitionKeyComparator.Mark endMark = null;
-          if (entry.getValue().endMarker != null) {
-            startMark = new PartitionKeyComparator.Mark(entry.getValue().endMarker.value,
-                entry.getValue().endMarker.isInclusive);
-          }
-          PartitionKeyComparator.Range range = new PartitionKeyComparator.Range(
-              entry.getKey(), startMark, endMark);
-          ranges.add(range);
-        }
-      }
-
-      if (ranges.isEmpty() && ops.isEmpty()) {
-        return null;
-      } else {
-        return new RowFilter(CompareFilter.CompareOp.EQUAL, new PartitionKeyComparator(
-            StringUtils.join(names, ","), StringUtils.join(HBaseUtils.getPartitionKeyTypes(parts), ","),
-            ranges, ops));
-      }
-    }
-
-    public void setStartMarker(String keyName, String keyType, String start, boolean isInclusive) {
-      if (markers.containsKey(keyName)) {
-        markers.get(keyName).startMarker = new ScanMarker(start, isInclusive, keyType);
-      } else {
-        ScanMarkerPair marker = new ScanMarkerPair(new ScanMarker(start, isInclusive, keyType), null);
-        markers.put(keyName, marker);
-      }
-    }
-
-    public ScanMarker getStartMarker(String keyName) {
-      if (markers.containsKey(keyName)) {
-        return markers.get(keyName).startMarker;
-      } else {
-        return null;
-      }
-    }
-
-    public void setEndMarker(String keyName, String keyType, String end, boolean isInclusive) {
-      if (markers.containsKey(keyName)) {
-        markers.get(keyName).endMarker = new ScanMarker(end, isInclusive, keyType);
-      } else {
-        ScanMarkerPair marker = new ScanMarkerPair(null, new ScanMarker(end, isInclusive, keyType));
-        markers.put(keyName, marker);
-      }
-    }
-
-    public ScanMarker getEndMarker(String keyName) {
-      if (markers.containsKey(keyName)) {
-        return markers.get(keyName).endMarker;
-      } else {
-        return null;
-      }
-    }
-
-    @Override
-    public FilterPlan and(FilterPlan other) {
-      List<ScanPlan> newSPlans = new ArrayList<ScanPlan>();
-      for (ScanPlan otherSPlan : other.getPlans()) {
-        newSPlans.add(this.and(otherSPlan));
-      }
-      return new MultiScanPlan(newSPlans);
-    }
-
-    private ScanPlan and(ScanPlan other) {
-      // create combined FilterPlan based on existing lhs and rhs plan
-      ScanPlan newPlan = new ScanPlan();
-      newPlan.markers.putAll(markers);
-
-      for (String keyName : other.markers.keySet()) {
-        if (newPlan.markers.containsKey(keyName)) {
-          // create new scan start
-          ScanMarker greaterStartMarker = getComparedMarker(this.getStartMarker(keyName),
-              other.getStartMarker(keyName), true);
-          if (greaterStartMarker != null) {
-            newPlan.setStartMarker(keyName, greaterStartMarker.type, greaterStartMarker.value, greaterStartMarker.isInclusive);
-          }
-
-          // create new scan end
-          ScanMarker lesserEndMarker = getComparedMarker(this.getEndMarker(keyName), other.getEndMarker(keyName),
-              false);
-          if (lesserEndMarker != null) {
-            newPlan.setEndMarker(keyName, lesserEndMarker.type, lesserEndMarker.value, lesserEndMarker.isInclusive);
-          }
-        } else {
-          newPlan.markers.put(keyName, other.markers.get(keyName));
-        }
-      }
-
-      newPlan.ops.addAll(ops);
-      newPlan.ops.addAll(other.ops);
-      return newPlan;
-    }
-
-    /**
-     * @param lStartMarker
-     * @param rStartMarker
-     * @param getGreater if true return greater startmarker, else return smaller one
-     * @return greater/lesser marker depending on value of getGreater
-     */
-    @VisibleForTesting
-    static ScanMarker getComparedMarker(ScanMarker lStartMarker, ScanMarker rStartMarker,
-        boolean getGreater) {
-      // if one of them has null bytes, just return other
-      if(lStartMarker == null) {
-        return rStartMarker;
-      } else if (rStartMarker == null) {
-        return lStartMarker;
-      }
-      TypeInfo expectedType =
-          TypeInfoUtils.getTypeInfoFromTypeString(lStartMarker.type);
-      ObjectInspector outputOI =
-          TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
-      Converter lConverter = ObjectInspectorConverters.getConverter(
-          PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
-      Converter rConverter = ObjectInspectorConverters.getConverter(
-          PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
-      Comparable lValue = (Comparable)lConverter.convert(lStartMarker.value);
-      Comparable rValue = (Comparable)rConverter.convert(rStartMarker.value);
-
-      int compareRes = lValue.compareTo(rValue);
-      if (compareRes == 0) {
-        // bytes are equal, now compare the isInclusive flags
-        if (lStartMarker.isInclusive == rStartMarker.isInclusive) {
-          // actually equal, so return any one
-          return lStartMarker;
-        }
-        boolean isInclusive = true;
-        // one that does not include the current bytes is greater
-        if (getGreater) {
-          isInclusive = false;
-        }
-        // else
-        return new ScanMarker(lStartMarker.value, isInclusive, lStartMarker.type);
-      }
-      if (getGreater) {
-        return compareRes == 1 ? lStartMarker : rStartMarker;
-      }
-      // else
-      return compareRes == -1 ? lStartMarker : rStartMarker;
-    }
-
-
-    @Override
-    public FilterPlan or(FilterPlan other) {
-      List<ScanPlan> plans = new ArrayList<ScanPlan>(getPlans());
-      plans.addAll(other.getPlans());
-      return new MultiScanPlan(plans);
-    }
-
-    @Override
-    public List<ScanPlan> getPlans() {
-      return Arrays.asList(this);
-    }
-
-
-    /**
-     * @return row suffix - This is appended to db + table, to generate start row for the Scan
-     */
-    public byte[] getStartRowSuffix(String dbName, String tableName, List<FieldSchema> parts) {
-      int majorPartsCount = getMajorPartsCount(parts);
-      List<String> majorPartTypes = new ArrayList<String>();
-      List<String> components = new ArrayList<String>();
-      boolean endPrefix = false;
-      for (int i=0;i<majorPartsCount;i++) {
-        majorPartTypes.add(parts.get(i).getType());
-        ScanMarker marker = markers.get(parts.get(i).getName()).startMarker;
-        if (marker != null) {
-          components.add(marker.value);
-          if (i==majorPartsCount-1) {
-            endPrefix = !marker.isInclusive;
-          }
-        } else {
-          components.add(null);
-          if (i==majorPartsCount-1) {
-            endPrefix = false;
-          }
-        }
-      }
-      byte[] bytes = HBaseUtils.buildPartitionKey(dbName, tableName, majorPartTypes, components, endPrefix);
-      return bytes;
-    }
-
-    /**
-     * @return row suffix - This is appended to db + table, to generate end row for the Scan
-     */
-    public byte[] getEndRowSuffix(String dbName, String tableName, List<FieldSchema> parts) {
-      int majorPartsCount = getMajorPartsCount(parts);
-      List<String> majorPartTypes = new ArrayList<String>();
-      List<String> components = new ArrayList<String>();
-      boolean endPrefix = false;
-      for (int i=0;i<majorPartsCount;i++) {
-        majorPartTypes.add(parts.get(i).getType());
-        ScanMarker marker = markers.get(parts.get(i).getName()).endMarker;
-        if (marker != null) {
-          components.add(marker.value);
-          if (i==majorPartsCount-1) {
-            endPrefix = marker.isInclusive;
-          }
-        } else {
-          components.add(null);
-          if (i==majorPartsCount-1) {
-            endPrefix = true;
-          }
-        }
-      }
-      byte[] bytes = HBaseUtils.buildPartitionKey(dbName, tableName, majorPartTypes, components, endPrefix);
-      if (components.isEmpty()) {
-        bytes[bytes.length-1]++;
-      }
-      return bytes;
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder();
-      sb.append("ScanPlan:\n");
-      for (Map.Entry<String, ScanMarkerPair> entry : markers.entrySet()) {
-        sb.append("key=" + entry.getKey() + "[startMarker=" + entry.getValue().startMarker
-            + ", endMarker=" + entry.getValue().endMarker + "]");
-      }
-      return sb.toString();
-    }
-
-  }
-
-  /**
-   * Visitor for ExpressionTree.
-   * It first generates the ScanPlan for the leaf nodes. The higher level nodes are
-   * either AND or OR operations. It then calls FilterPlan.and and FilterPlan.or with
-   * the child nodes to generate the plans for higher level nodes.
-   */
-  @VisibleForTesting
-  static class PartitionFilterGenerator extends TreeVisitor {
-    private FilterPlan curPlan;
-
-    // this tells us if there is a condition that did not get included in the plan
-    // such condition would be treated as getting evaluated to TRUE
-    private boolean hasUnsupportedCondition = false;
-
-    //Need to cache the left plans for the TreeNode. Use IdentityHashMap here
-    // as we don't want to dedupe on two TreeNode that are otherwise considered equal
-    Map<TreeNode, FilterPlan> leftPlans = new IdentityHashMap<TreeNode, FilterPlan>();
-
-    // temporary params for current left and right side plans, for AND, OR
-    private FilterPlan rPlan;
-
-    private Map<String, String> nameToType = new HashMap<String, String>();
-
-    public PartitionFilterGenerator(List<FieldSchema> parts) {
-      for (FieldSchema part : parts) {
-        nameToType.put(part.getName(), part.getType());
-      }
-    }
-
-    FilterPlan getPlan() {
-      return curPlan;
-    }
-
-    @Override
-    protected void beginTreeNode(TreeNode node) throws MetaException {
-      // reset the params
-      curPlan = rPlan = null;
-    }
-
-    @Override
-    protected void midTreeNode(TreeNode node) throws MetaException {
-      leftPlans.put(node, curPlan);
-      curPlan = null;
-    }
-
-    @Override
-    protected void endTreeNode(TreeNode node) throws MetaException {
-      rPlan = curPlan;
-      FilterPlan lPlan = leftPlans.get(node);
-      leftPlans.remove(node);
-
-      switch (node.getAndOr()) {
-      case AND:
-        curPlan = lPlan.and(rPlan);
-        break;
-      case OR:
-        curPlan = lPlan.or(rPlan);
-        break;
-      default:
-        throw new AssertionError("Unexpected logical operation " + node.getAndOr());
-      }
-
-    }
-
-
-    @Override
-    public void visit(LeafNode node) throws MetaException {
-      ScanPlan leafPlan = new ScanPlan();
-      curPlan = leafPlan;
-
-      // this is a condition on first partition column, so might influence the
-      // start and end of the scan
-      final boolean INCLUSIVE = true;
-      switch (node.operator) {
-      case EQUALS:
-        leafPlan.setStartMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), INCLUSIVE);
-        leafPlan.setEndMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), INCLUSIVE);
-        break;
-      case GREATERTHAN:
-        leafPlan.setStartMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), !INCLUSIVE);
-        break;
-      case GREATERTHANOREQUALTO:
-        leafPlan.setStartMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), INCLUSIVE);
-        break;
-      case LESSTHAN:
-        leafPlan.setEndMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), !INCLUSIVE);
-        break;
-      case LESSTHANOREQUALTO:
-        leafPlan.setEndMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), INCLUSIVE);
-        break;
-      case LIKE:
-        leafPlan.ops.add(new Operator(Operator.Type.LIKE, node.keyName, node.value.toString()));
-        break;
-      case NOTEQUALS:
-      case NOTEQUALS2:
-        leafPlan.ops.add(new Operator(Operator.Type.NOTEQUALS, node.keyName, node.value.toString()));
-        break;
-      }
-    }
-
-    private boolean hasUnsupportedCondition() {
-      return hasUnsupportedCondition;
-    }
-
-  }
-
-  public static class PlanResult {
-    public final FilterPlan plan;
-    public final boolean hasUnsupportedCondition;
-    PlanResult(FilterPlan plan, boolean hasUnsupportedCondition) {
-      this.plan = plan;
-      this.hasUnsupportedCondition = hasUnsupportedCondition;
-    }
-  }
-
-  public static PlanResult getFilterPlan(ExpressionTree exprTree, List<FieldSchema> parts) throws MetaException {
-    if (exprTree == null) {
-      // TODO: if exprTree is null, we should do what ObjectStore does. See HIVE-10102
-      return new PlanResult(new ScanPlan(), true);
-    }
-    PartitionFilterGenerator pGenerator = new PartitionFilterGenerator(parts);
-    exprTree.accept(pGenerator);
-    return new PlanResult(pGenerator.getPlan(), pGenerator.hasUnsupportedCondition());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseImport.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseImport.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseImport.java
deleted file mode 100644
index 5f89769..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseImport.java
+++ /dev/null
@@ -1,619 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.Deadline;
-import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A tool to take the contents of an RDBMS based Hive metastore and import it into an HBase based
- * one.  To use this the config files for Hive configured to work with the RDBMS (that is,
- * including the JDBC string, etc.) as well as HBase configuration files must be in the path.
- * There should not be a hive-site.xml that specifies HBaseStore in the path.  This tool will then
- * handle connecting to the RDBMS via the {@link org.apache.hadoop.hive.metastore.ObjectStore}
- * and HBase via {@link org.apache.hadoop.hive.metastore.hbase.HBaseStore} and transferring the
- * data.
- *
- * This tool can import an entire metastore or only selected objects.  When selecting objects it
- * is necessary to fully specify the object's name.  For example, if you want to import the table
- * T in the default database it needs to be identified as default.T.  The same is true for
- * functions.  When an object is specified, everything under that object will be imported (e.g.
- * if you select database D, then all tables and functions in that database will be
- * imported as well).
- *
- * At this point only tables and partitions are handled in parallel as it is assumed there are
- * relatively few of everything else.
- *
- * Note that HBaseSchemaTool must have already been used to create the appropriate tables in HBase.
- */
-public class HBaseImport {
-
-  static final private Logger LOG = LoggerFactory.getLogger(HBaseImport.class.getName());
-
-  public static int main(String[] args) {
-    try {
-      HBaseImport tool = new HBaseImport();
-      int rv = tool.init(args);
-      if (rv != 0) return rv;
-      tool.run();
-    } catch (Exception e) {
-      System.err.println("Caught exception " + e.getClass().getName() + " with message <" +
-          e.getMessage() + ">");
-      return 1;
-    }
-    return 0;
-  }
-
-  private ThreadLocal<RawStore> rdbmsStore = new ThreadLocal<RawStore>() {
-    @Override
-    protected RawStore initialValue() {
-      if (rdbmsConf == null) {
-        throw new RuntimeException("order violation, need to set rdbms conf first");
-      }
-      RawStore os = new ObjectStore();
-      os.setConf(rdbmsConf);
-      return os;
-    }
-  };
-
-  private ThreadLocal<RawStore> hbaseStore = new ThreadLocal<RawStore>() {
-    @Override
-    protected RawStore initialValue() {
-      if (hbaseConf == null) {
-        throw new RuntimeException("order violation, need to set hbase conf first");
-      }
-      RawStore hs = new HBaseStore();
-      hs.setConf(hbaseConf);
-      return hs;
-    }
-  };
-
-  private Configuration rdbmsConf;
-  private Configuration hbaseConf;
-  private List<Database> dbs;
-  private BlockingQueue<Table> partitionedTables;
-  private BlockingQueue<String[]> tableNameQueue;
-  private BlockingQueue<String[]> indexNameQueue;
-  private BlockingQueue<PartQueueEntry> partQueue;
-  private boolean writingToQueue, readersFinished;
-  private boolean doKerberos, doAll;
-  private List<String> rolesToImport, dbsToImport, tablesToImport, functionsToImport;
-  private int parallel;
-  private int batchSize;
-
-  private HBaseImport() {}
-
-  @VisibleForTesting
-  public HBaseImport(String... args) throws ParseException {
-    init(args);
-  }
-
-  private int init(String... args) throws ParseException {
-    Options options = new Options();
-
-    doAll = doKerberos = false;
-    parallel = 1;
-    batchSize = 1000;
-
-    options.addOption(OptionBuilder
-        .withLongOpt("all")
-        .withDescription("Import the full metastore")
-        .create('a'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("batchsize")
-        .withDescription("Number of partitions to read and write in a batch, defaults to 1000")
-            .hasArg()
-            .create('b'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("database")
-        .withDescription("Import a single database")
-        .hasArgs()
-        .create('d'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("help")
-        .withDescription("You're looking at it")
-        .create('h'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("function")
-        .withDescription("Import a single function")
-        .hasArgs()
-        .create('f'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("kerberos")
-        .withDescription("Import all kerberos related objects (master key, tokens)")
-        .create('k'));
-
-     options.addOption(OptionBuilder
-        .withLongOpt("parallel")
-        .withDescription("Parallel factor for loading (only applied to tables and partitions), " +
-            "defaults to 1")
-        .hasArg()
-        .create('p'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("role")
-        .withDescription("Import a single role")
-        .hasArgs()
-        .create('r'));
-
-   options.addOption(OptionBuilder
-        .withLongOpt("tables")
-        .withDescription("Import a single tables")
-        .hasArgs()
-        .create('t'));
-
-    CommandLine cli = new GnuParser().parse(options, args);
-
-    // Process help, if it was asked for, this must be done first
-    if (cli.hasOption('h')) {
-      printHelp(options);
-      return 1;
-    }
-
-    boolean hasCmd = false;
-    // Now process the other command line args
-    if (cli.hasOption('a')) {
-      hasCmd = true;
-      doAll = true;
-    }
-    if (cli.hasOption('b')) {
-      batchSize = Integer.parseInt(cli.getOptionValue('b'));
-    }
-    if (cli.hasOption('d')) {
-      hasCmd = true;
-      dbsToImport = Arrays.asList(cli.getOptionValues('d'));
-    }
-    if (cli.hasOption('f')) {
-      hasCmd = true;
-      functionsToImport = Arrays.asList(cli.getOptionValues('f'));
-    }
-    if (cli.hasOption('p')) {
-      parallel = Integer.parseInt(cli.getOptionValue('p'));
-    }
-    if (cli.hasOption('r')) {
-      hasCmd = true;
-      rolesToImport = Arrays.asList(cli.getOptionValues('r'));
-    }
-    if (cli.hasOption('k')) {
-      doKerberos = true;
-    }
-    if (cli.hasOption('t')) {
-      hasCmd = true;
-      tablesToImport = Arrays.asList(cli.getOptionValues('t'));
-    }
-    if (!hasCmd) {
-      printHelp(options);
-      return 1;
-    }
-
-    dbs = new ArrayList<>();
-    // We don't want to bound the size of the table queue because we keep it all in memory
-    partitionedTables = new LinkedBlockingQueue<>();
-    tableNameQueue = new LinkedBlockingQueue<>();
-    indexNameQueue = new LinkedBlockingQueue<>();
-
-    // Bound the size of this queue so we don't get too much in memory.
-    partQueue = new ArrayBlockingQueue<>(parallel * 2);
-    return 0;
-  }
-
-  private void printHelp(Options options) {
-    (new HelpFormatter()).printHelp("hbaseschematool", options);
-  }
-
-  @VisibleForTesting
-  void run() throws MetaException, InstantiationException, IllegalAccessException,
-      NoSuchObjectException, InvalidObjectException, InterruptedException {
-    // Order here is crucial, as you can't add tables until you've added databases, etc.
-    init();
-    if (doAll || rolesToImport != null) {
-      copyRoles();
-    }
-    if (doAll || dbsToImport != null) {
-      copyDbs();
-    }
-    if (doAll || dbsToImport != null || tablesToImport != null) {
-      copyTables();
-      copyPartitions();
-      copyIndexes();
-    }
-    if (doAll || dbsToImport != null || functionsToImport != null) {
-      copyFunctions();
-    }
-    if (doAll || doKerberos) {
-      copyKerberos();
-    }
-  }
-
-  private void init() throws MetaException, IllegalAccessException, InstantiationException {
-    if (rdbmsConf != null) {
-      // We've been configured for testing, so don't do anything here.
-      return;
-    }
-    // We're depending on having everything properly in the path
-    rdbmsConf = new HiveConf();
-    hbaseConf = new HiveConf();//
-    HiveConf.setVar(hbaseConf, HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
-        HBaseStore.class.getName());
-    HiveConf.setBoolVar(hbaseConf, HiveConf.ConfVars.METASTORE_FASTPATH, true);
-
-    // First get a connection to the RDBMS based store
-    rdbmsStore.get().setConf(rdbmsConf);
-
-    // Get a connection to the HBase based store
-    hbaseStore.get().setConf(hbaseConf);
-  }
-
-  private void copyRoles() throws NoSuchObjectException, InvalidObjectException, MetaException {
-    screen("Copying roles");
-    List<String> toCopy = doAll ? rdbmsStore.get().listRoleNames() : rolesToImport;
-    for (String roleName : toCopy) {
-      Role role = rdbmsStore.get().getRole(roleName);
-      screen("Copying role " + roleName);
-      hbaseStore.get().addRole(roleName, role.getOwnerName());
-    }
-  }
-
-  private void copyDbs() throws MetaException, NoSuchObjectException, InvalidObjectException {
-    screen("Copying databases");
-    List<String> toCopy = doAll ? rdbmsStore.get().getAllDatabases() : dbsToImport;
-    for (String dbName : toCopy) {
-      Database db = rdbmsStore.get().getDatabase(dbName);
-      dbs.add(db);
-      screen("Copying database " + dbName);
-      hbaseStore.get().createDatabase(db);
-    }
-  }
-
-  private void copyTables() throws MetaException, InvalidObjectException, InterruptedException {
-    screen("Copying tables");
-
-    // Start the parallel threads that will copy the tables
-    Thread[] copiers = new Thread[parallel];
-    writingToQueue = true;
-    for (int i = 0; i < parallel; i++) {
-      copiers[i] = new TableCopier();
-      copiers[i].start();
-    }
-
-    // Put tables from the databases we copied into the queue
-    for (Database db : dbs) {
-      screen("Coyping tables in database " + db.getName());
-      for (String tableName : rdbmsStore.get().getAllTables(db.getName())) {
-        tableNameQueue.put(new String[]{db.getName(), tableName});
-      }
-    }
-
-    // Now put any specifically requested tables into the queue
-    if (tablesToImport != null) {
-      for (String compoundTableName : tablesToImport) {
-        String[] tn = compoundTableName.split("\\.");
-        if (tn.length != 2) {
-          error(compoundTableName + " not in proper form.  Must be in form dbname.tablename.  " +
-              "Ignoring this table and continuing.");
-        } else {
-          tableNameQueue.put(new String[]{tn[0], tn[1]});
-        }
-      }
-    }
-    writingToQueue = false;
-
-    // Wait until we've finished adding all the tables
-    for (Thread copier : copiers) copier.join();
- }
-
-  private class TableCopier extends Thread {
-    @Override
-    public void run() {
-      while (writingToQueue || tableNameQueue.size() > 0) {
-        try {
-          String[] name = tableNameQueue.poll(1, TimeUnit.SECONDS);
-          if (name != null) {
-            Table table = rdbmsStore.get().getTable(name[0], name[1]);
-            // If this has partitions, put it in the list to fetch partions for
-            if (table.getPartitionKeys() != null && table.getPartitionKeys().size() > 0) {
-              partitionedTables.put(table);
-            }
-            screen("Copying table " + name[0] + "." + name[1]);
-            hbaseStore.get().createTable(table);
-
-            // See if the table has any constraints, and if so copy those as well
-            List<SQLPrimaryKey> pk =
-                rdbmsStore.get().getPrimaryKeys(table.getDbName(), table.getTableName());
-            if (pk != null && pk.size() > 0) {
-              LOG.debug("Found primary keys, adding them");
-              hbaseStore.get().addPrimaryKeys(pk);
-            }
-
-            // Passing null as the target table name results in all of the foreign keys being
-            // retrieved.
-            List<SQLForeignKey> fks =
-                rdbmsStore.get().getForeignKeys(null, null, table.getDbName(), table.getTableName());
-            if (fks != null && fks.size() > 0) {
-              LOG.debug("Found foreign keys, adding them");
-              hbaseStore.get().addForeignKeys(fks);
-            }
-          }
-        } catch (InterruptedException | MetaException | InvalidObjectException e) {
-          throw new RuntimeException(e);
-        }
-      }
-    }
-  }
-
-  private void copyIndexes() throws MetaException, InvalidObjectException, InterruptedException {
-    screen("Copying indexes");
-
-    // Start the parallel threads that will copy the indexes
-    Thread[] copiers = new Thread[parallel];
-    writingToQueue = true;
-    for (int i = 0; i < parallel; i++) {
-      copiers[i] = new IndexCopier();
-      copiers[i].start();
-    }
-
-    // Put indexes from the databases we copied into the queue
-    for (Database db : dbs) {
-      screen("Coyping indexes in database " + db.getName());
-      for (String tableName : rdbmsStore.get().getAllTables(db.getName())) {
-        for (Index index : rdbmsStore.get().getIndexes(db.getName(), tableName, -1)) {
-          indexNameQueue.put(new String[]{db.getName(), tableName, index.getIndexName()});
-        }
-      }
-    }
-
-    // Now put any specifically requested tables into the queue
-    if (tablesToImport != null) {
-      for (String compoundTableName : tablesToImport) {
-        String[] tn = compoundTableName.split("\\.");
-        if (tn.length != 2) {
-          error(compoundTableName + " not in proper form.  Must be in form dbname.tablename.  " +
-              "Ignoring this table and continuing.");
-        } else {
-          for (Index index : rdbmsStore.get().getIndexes(tn[0], tn[1], -1)) {
-            indexNameQueue.put(new String[]{tn[0], tn[1], index.getIndexName()});
-          }
-        }
-      }
-    }
-
-    writingToQueue = false;
-
-    // Wait until we've finished adding all the tables
-    for (Thread copier : copiers) copier.join();
- }
-
-  private class IndexCopier extends Thread {
-    @Override
-    public void run() {
-      while (writingToQueue || indexNameQueue.size() > 0) {
-        try {
-          String[] name = indexNameQueue.poll(1, TimeUnit.SECONDS);
-          if (name != null) {
-            Index index = rdbmsStore.get().getIndex(name[0], name[1], name[2]);
-            screen("Copying index " + name[0] + "." + name[1] + "." + name[2]);
-            hbaseStore.get().addIndex(index);
-          }
-        } catch (InterruptedException | MetaException | InvalidObjectException e) {
-          throw new RuntimeException(e);
-        }
-      }
-    }
-  }
-
-  /* Partition copying is a little complex.  As we went through and copied the tables we put each
-   * partitioned table into a queue.  We will now go through that queue and add partitions for the
-   * tables.  We do the finding of partitions and writing of them separately and in parallel.
-   * This way if there is one table with >> partitions then all of the others that skew won't
-   * hurt us.  To avoid pulling all of the partitions for a table into memory, we batch up
-   * partitions (by default in batches of 1000) and copy them over in batches.
-   */
-  private void copyPartitions() throws MetaException, NoSuchObjectException,
-      InvalidObjectException, InterruptedException {
-    screen("Copying partitions");
-    readersFinished = false;
-    Thread[] readers = new Thread[parallel];
-    Thread[] writers = new Thread[parallel];
-    for (int i = 0; i < parallel; i++) {
-      readers[i] = new PartitionReader();
-      readers[i].start();
-      writers[i] = new PartitionWriter();
-      writers[i].start();
-    }
-
-    for (Thread reader : readers) reader.join();
-    readersFinished = true;
-
-    // Wait until we've finished adding all the partitions
-    for (Thread writer : writers) writer.join();
-  }
-
-  private class PartitionReader extends Thread {
-    @Override
-    public void run() {
-      while (partitionedTables.size() > 0) {
-        try {
-          Table table = partitionedTables.poll(1, TimeUnit.SECONDS);
-          if (table != null) {
-            screen("Fetching partitions for table " + table.getDbName() + "." +
-                table.getTableName());
-            List<String> partNames =
-                rdbmsStore.get().listPartitionNames(table.getDbName(), table.getTableName(),
-                    (short) -1);
-            if (partNames.size() <= batchSize) {
-              LOG.debug("Adding all partition names to queue for " + table.getDbName() + "." +
-                  table.getTableName());
-              partQueue.put(new PartQueueEntry(table.getDbName(), table.getTableName(), partNames));
-            } else {
-              int goUntil = partNames.size() % batchSize == 0 ? partNames.size() / batchSize :
-                  partNames.size() / batchSize + 1;
-              for (int i = 0; i < goUntil; i++) {
-                int start = i * batchSize;
-                int end = Math.min((i + 1) * batchSize, partNames.size());
-                LOG.debug("Adding partitions " + start + " to " + end + " for " + table.getDbName()
-                    + "." + table.getTableName());
-                partQueue.put(new PartQueueEntry(table.getDbName(), table.getTableName(),
-                    partNames.subList(start, end)));
-              }
-            }
-          }
-        } catch (InterruptedException | MetaException e) {
-          throw new RuntimeException(e);
-        }
-      }
-    }
-  }
-
-  private class PartitionWriter extends Thread {
-    @Override
-    public void run() {
-      // This keeps us from throwing exceptions in our raw store calls
-      Deadline.registerIfNot(1000000);
-      while (!readersFinished || partQueue.size() > 0) {
-        try {
-          PartQueueEntry entry = partQueue.poll(1, TimeUnit.SECONDS);
-          if (entry != null) {
-            LOG.info("Writing partitions " + entry.dbName + "." + entry.tableName + "." +
-                StringUtils.join(entry.partNames, ':'));
-            // Fetch these partitions and write them to HBase
-            Deadline.startTimer("hbaseimport");
-            List<Partition> parts =
-                rdbmsStore.get().getPartitionsByNames(entry.dbName, entry.tableName,
-                    entry.partNames);
-            hbaseStore.get().addPartitions(entry.dbName, entry.tableName, parts);
-            Deadline.stopTimer();
-          }
-        } catch (InterruptedException | MetaException | InvalidObjectException |
-            NoSuchObjectException e) {
-          throw new RuntimeException(e);
-        }
-      }
-    }
-  }
-
-  private void copyFunctions() throws MetaException, NoSuchObjectException, InvalidObjectException {
-    screen("Copying functions");
-    // Copy any functions from databases we copied.
-    for (Database db : dbs) {
-      screen("Copying functions in database " + db.getName());
-      for (String funcName : rdbmsStore.get().getFunctions(db.getName(), "*")) {
-        copyOneFunction(db.getName(), funcName);
-      }
-    }
-    // Now do any specifically requested functions
-    if (functionsToImport != null) {
-      for (String compoundFuncName : functionsToImport) {
-        String[] fn = compoundFuncName.split("\\.");
-        if (fn.length != 2) {
-          error(compoundFuncName + " not in proper form.  Must be in form dbname.funcname.  " +
-              "Ignoring this function and continuing.");
-        } else {
-          copyOneFunction(fn[0], fn[1]);
-        }
-      }
-    }
-  }
-
-  private void copyOneFunction(String dbName, String funcName) throws MetaException,
-      InvalidObjectException {
-    Function func = rdbmsStore.get().getFunction(dbName, funcName);
-    screen("Copying function " + dbName + "." + funcName);
-    hbaseStore.get().createFunction(func);
-  }
-
-  private void copyKerberos() throws MetaException {
-    screen("Copying kerberos related items");
-    for (String tokenId : rdbmsStore.get().getAllTokenIdentifiers()) {
-      String token = rdbmsStore.get().getToken(tokenId);
-      hbaseStore.get().addToken(tokenId, token);
-    }
-    for (String masterKey : rdbmsStore.get().getMasterKeys()) {
-      hbaseStore.get().addMasterKey(masterKey);
-    }
-  }
-
-  private void screen(String msg) {
-    LOG.info(msg);
-    System.out.println(msg);
-  }
-
-  private void error(String msg) {
-    LOG.error(msg);
-    System.err.println("ERROR:  " + msg);
-  }
-
-  @VisibleForTesting
-  void setConnections(RawStore rdbms, RawStore hbase) {
-    rdbmsStore.set(rdbms);
-    hbaseStore.set(hbase);
-    rdbmsConf = rdbms.getConf();
-    hbaseConf = hbase.getConf();
-  }
-
-  private static class PartQueueEntry {
-    final String dbName;
-    final String tableName;
-    final List<String> partNames;
-
-    PartQueueEntry(String d, String t, List<String> p) {
-      dbName = d;
-      tableName = t;
-      partNames = p;
-    }
-  }
-
-}


[05/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
deleted file mode 100644
index e55c412..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implements
-    IExtrapolatePartStatus {
-
-  private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
-
-  @Override
-  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
-      List<ColumnStatistics> css) throws MetaException {
-    ColumnStatisticsObj statsObj = null;
-
-    // check if all the ColumnStatisticsObjs contain stats and all the ndv are
-    // bitvectors
-    boolean doAllPartitionContainStats = partNames.size() == css.size();
-    LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
-    NumDistinctValueEstimator ndvEstimator = null;
-    String colType = null;
-    for (ColumnStatistics cs : css) {
-      if (cs.getStatsObjSize() != 1) {
-        throw new MetaException(
-            "The number of columns should be exactly one in aggrStats, but found "
-                + cs.getStatsObjSize());
-      }
-      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-      if (statsObj == null) {
-        colType = cso.getColType();
-        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
-            .getStatsData().getSetField());
-      }
-      if (!cso.getStatsData().getDoubleStats().isSetBitVectors()
-          || cso.getStatsData().getDoubleStats().getBitVectors().length() == 0) {
-        ndvEstimator = null;
-        break;
-      } else {
-        // check if all of the bit vectors can merge
-        NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
-            .getNumDistinctValueEstimator(cso.getStatsData().getDoubleStats().getBitVectors());
-        if (ndvEstimator == null) {
-          ndvEstimator = estimator;
-        } else {
-          if (ndvEstimator.canMerge(estimator)) {
-            continue;
-          } else {
-            ndvEstimator = null;
-            break;
-          }
-        }
-      }
-    }
-    if (ndvEstimator != null) {
-      ndvEstimator = NumDistinctValueEstimatorFactory
-          .getEmptyNumDistinctValueEstimator(ndvEstimator);
-    }
-    LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
-    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
-    if (doAllPartitionContainStats || css.size() < 2) {
-      DoubleColumnStatsData aggregateData = null;
-      long lowerBound = 0;
-      long higherBound = 0;
-      double densityAvgSum = 0.0;
-      for (ColumnStatistics cs : css) {
-        ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-        DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
-        lowerBound = Math.max(lowerBound, newData.getNumDVs());
-        higherBound += newData.getNumDVs();
-        densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
-        if (ndvEstimator != null) {
-          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
-              .getNumDistinctValueEstimator(newData.getBitVectors()));
-        }
-        if (aggregateData == null) {
-          aggregateData = newData.deepCopy();
-        } else {
-          aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
-          aggregateData
-              .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
-          aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-          aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
-        }
-      }
-      if (ndvEstimator != null) {
-        // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
-        // use uniform distribution assumption because we can merge bitvectors
-        // to get a good estimation.
-        aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
-      } else {
-        long estimation;
-        if (useDensityFunctionForNDVEstimation) {
-          // We have estimation, lowerbound and higherbound. We use estimation
-          // if it is between lowerbound and higherbound.
-          double densityAvg = densityAvgSum / partNames.size();
-          estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
-          if (estimation < lowerBound) {
-            estimation = lowerBound;
-          } else if (estimation > higherBound) {
-            estimation = higherBound;
-          }
-        } else {
-          estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
-        }
-        aggregateData.setNumDVs(estimation);
-      }
-      columnStatisticsData.setDoubleStats(aggregateData);
-    } else {
-      // we need extrapolation
-      LOG.debug("start extrapolation for " + colName);
-      Map<String, Integer> indexMap = new HashMap<String, Integer>();
-      for (int index = 0; index < partNames.size(); index++) {
-        indexMap.put(partNames.get(index), index);
-      }
-      Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
-      Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
-      // while we scan the css, we also get the densityAvg, lowerbound and
-      // higerbound when useDensityFunctionForNDVEstimation is true.
-      double densityAvgSum = 0.0;
-      if (ndvEstimator == null) {
-        // if not every partition uses bitvector for ndv, we just fall back to
-        // the traditional extrapolation methods.
-        for (ColumnStatistics cs : css) {
-          String partName = cs.getStatsDesc().getPartName();
-          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-          DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
-          if (useDensityFunctionForNDVEstimation) {
-            densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
-          }
-          adjustedIndexMap.put(partName, (double) indexMap.get(partName));
-          adjustedStatsMap.put(partName, cso.getStatsData());
-        }
-      } else {
-        // we first merge all the adjacent bitvectors that we could merge and
-        // derive new partition names and index.
-        StringBuilder pseudoPartName = new StringBuilder();
-        double pseudoIndexSum = 0;
-        int length = 0;
-        int curIndex = -1;
-        DoubleColumnStatsData aggregateData = null;
-        for (ColumnStatistics cs : css) {
-          String partName = cs.getStatsDesc().getPartName();
-          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-          DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
-          // newData.isSetBitVectors() should be true for sure because we
-          // already checked it before.
-          if (indexMap.get(partName) != curIndex) {
-            // There is bitvector, but it is not adjacent to the previous ones.
-            if (length > 0) {
-              // we have to set ndv
-              adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
-              aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
-              ColumnStatisticsData csd = new ColumnStatisticsData();
-              csd.setDoubleStats(aggregateData);
-              adjustedStatsMap.put(pseudoPartName.toString(), csd);
-              if (useDensityFunctionForNDVEstimation) {
-                densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
-              }
-              // reset everything
-              pseudoPartName = new StringBuilder();
-              pseudoIndexSum = 0;
-              length = 0;
-              ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
-            }
-            aggregateData = null;
-          }
-          curIndex = indexMap.get(partName);
-          pseudoPartName.append(partName);
-          pseudoIndexSum += curIndex;
-          length++;
-          curIndex++;
-          if (aggregateData == null) {
-            aggregateData = newData.deepCopy();
-          } else {
-            aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
-            aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
-                newData.getHighValue()));
-            aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-          }
-          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
-              .getNumDistinctValueEstimator(newData.getBitVectors()));
-        }
-        if (length > 0) {
-          // we have to set ndv
-          adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
-          aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
-          ColumnStatisticsData csd = new ColumnStatisticsData();
-          csd.setDoubleStats(aggregateData);
-          adjustedStatsMap.put(pseudoPartName.toString(), csd);
-          if (useDensityFunctionForNDVEstimation) {
-            densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
-          }
-        }
-      }
-      extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
-          adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
-    }
-    LOG.debug("Ndv estimatation for " + colName + " is "
-        + columnStatisticsData.getDoubleStats().getNumDVs());
-    statsObj.setStatsData(columnStatisticsData);
-    return statsObj;
-  }
-
-  @Override
-  public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
-      int numPartsWithStats, Map<String, Double> adjustedIndexMap,
-      Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
-    int rightBorderInd = numParts;
-    DoubleColumnStatsData extrapolateDoubleData = new DoubleColumnStatsData();
-    Map<String, DoubleColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
-    for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
-      extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDoubleStats());
-    }
-    List<Map.Entry<String, DoubleColumnStatsData>> list = new LinkedList<Map.Entry<String, DoubleColumnStatsData>>(
-        extractedAdjustedStatsMap.entrySet());
-    // get the lowValue
-    Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
-      public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
-          Map.Entry<String, DoubleColumnStatsData> o2) {
-        return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
-      }
-    });
-    double minInd = adjustedIndexMap.get(list.get(0).getKey());
-    double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
-    double lowValue = 0;
-    double min = list.get(0).getValue().getLowValue();
-    double max = list.get(list.size() - 1).getValue().getLowValue();
-    if (minInd == maxInd) {
-      lowValue = min;
-    } else if (minInd < maxInd) {
-      // left border is the min
-      lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
-    } else {
-      // right border is the min
-      lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
-    }
-
-    // get the highValue
-    Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
-      public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
-          Map.Entry<String, DoubleColumnStatsData> o2) {
-        return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
-      }
-    });
-    minInd = adjustedIndexMap.get(list.get(0).getKey());
-    maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
-    double highValue = 0;
-    min = list.get(0).getValue().getHighValue();
-    max = list.get(list.size() - 1).getValue().getHighValue();
-    if (minInd == maxInd) {
-      highValue = min;
-    } else if (minInd < maxInd) {
-      // right border is the max
-      highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
-    } else {
-      // left border is the max
-      highValue = (min + (max - min) * minInd / (minInd - maxInd));
-    }
-
-    // get the #nulls
-    long numNulls = 0;
-    for (Map.Entry<String, DoubleColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
-      numNulls += entry.getValue().getNumNulls();
-    }
-    // we scale up sumNulls based on the number of partitions
-    numNulls = numNulls * numParts / numPartsWithStats;
-
-    // get the ndv
-    long ndv = 0;
-    long ndvMin = 0;
-    long ndvMax = 0;
-    Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
-      public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
-          Map.Entry<String, DoubleColumnStatsData> o2) {
-        return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
-      }
-    });
-    long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
-    long higherBound = 0;
-    for (Map.Entry<String, DoubleColumnStatsData> entry : list) {
-      higherBound += entry.getValue().getNumDVs();
-    }
-    if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
-      ndv = (long) ((highValue - lowValue) / densityAvg);
-      if (ndv < lowerBound) {
-        ndv = lowerBound;
-      } else if (ndv > higherBound) {
-        ndv = higherBound;
-      }
-    } else {
-      minInd = adjustedIndexMap.get(list.get(0).getKey());
-      maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
-      ndvMin = list.get(0).getValue().getNumDVs();
-      ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
-      if (minInd == maxInd) {
-        ndv = ndvMin;
-      } else if (minInd < maxInd) {
-        // right border is the max
-        ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
-      } else {
-        // left border is the max
-        ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
-      }
-    }
-    extrapolateDoubleData.setLowValue(lowValue);
-    extrapolateDoubleData.setHighValue(highValue);
-    extrapolateDoubleData.setNumNulls(numNulls);
-    extrapolateDoubleData.setNumDVs(ndv);
-    extrapolateData.setDoubleStats(extrapolateDoubleData);
-  }
-  
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
deleted file mode 100644
index acf679e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Map;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-
-public interface IExtrapolatePartStatus {
-  // The following function will extrapolate the stats when the column stats of
-  // some partitions are missing.
-  /**
-   * @param extrapolateData
-   *          it will carry back the specific stats, e.g., DOUBLE_STATS or
-   *          LONG_STATS
-   * @param numParts
-   *          the total number of partitions
-   * @param numPartsWithStats
-   *          the number of partitions that have stats
-   * @param adjustedIndexMap
-   *          the partition name to index map
-   * @param adjustedStatsMap
-   *          the partition name to its stats map
-   * @param densityAvg
-   *          the average of ndv density, which is useful when
-   *          useDensityFunctionForNDVEstimation is true.
-   */
-  public abstract void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
-      int numPartsWithStats, Map<String, Double> adjustedIndexMap,
-      Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg);
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
deleted file mode 100644
index 2ee09f3..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
-    IExtrapolatePartStatus {
-
-  private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
-
-  @Override
-  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
-      List<ColumnStatistics> css) throws MetaException {
-    ColumnStatisticsObj statsObj = null;
-
-    // check if all the ColumnStatisticsObjs contain stats and all the ndv are
-    // bitvectors
-    boolean doAllPartitionContainStats = partNames.size() == css.size();
-    LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
-    NumDistinctValueEstimator ndvEstimator = null;
-    String colType = null;
-    for (ColumnStatistics cs : css) {
-      if (cs.getStatsObjSize() != 1) {
-        throw new MetaException(
-            "The number of columns should be exactly one in aggrStats, but found "
-                + cs.getStatsObjSize());
-      }
-      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-      if (statsObj == null) {
-        colType = cso.getColType();
-        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
-            .getStatsData().getSetField());
-      }
-      if (!cso.getStatsData().getLongStats().isSetBitVectors()
-          || cso.getStatsData().getLongStats().getBitVectors().length() == 0) {
-        ndvEstimator = null;
-        break;
-      } else {
-        // check if all of the bit vectors can merge
-        NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
-            .getNumDistinctValueEstimator(cso.getStatsData().getLongStats().getBitVectors());
-        if (ndvEstimator == null) {
-          ndvEstimator = estimator;
-        } else {
-          if (ndvEstimator.canMerge(estimator)) {
-            continue;
-          } else {
-            ndvEstimator = null;
-            break;
-          }
-        }
-      }
-    }
-    if (ndvEstimator != null) {
-      ndvEstimator = NumDistinctValueEstimatorFactory
-          .getEmptyNumDistinctValueEstimator(ndvEstimator);
-    }
-    LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
-    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
-    if (doAllPartitionContainStats || css.size() < 2) {
-      LongColumnStatsData aggregateData = null;
-      long lowerBound = 0;
-      long higherBound = 0;
-      double densityAvgSum = 0.0;
-      for (ColumnStatistics cs : css) {
-        ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-        LongColumnStatsData newData = cso.getStatsData().getLongStats();
-        lowerBound = Math.max(lowerBound, newData.getNumDVs());
-        higherBound += newData.getNumDVs();
-        densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
-        if (ndvEstimator != null) {
-          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
-              .getNumDistinctValueEstimator(newData.getBitVectors()));
-        }
-        if (aggregateData == null) {
-          aggregateData = newData.deepCopy();
-        } else {
-          aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
-          aggregateData
-              .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
-          aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-          aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
-        }
-      }
-      if (ndvEstimator != null) {
-        // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
-        // use uniform distribution assumption because we can merge bitvectors
-        // to get a good estimation.
-        aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
-      } else {
-        long estimation;
-        if (useDensityFunctionForNDVEstimation) {
-          // We have estimation, lowerbound and higherbound. We use estimation
-          // if it is between lowerbound and higherbound.
-          double densityAvg = densityAvgSum / partNames.size();
-          estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
-          if (estimation < lowerBound) {
-            estimation = lowerBound;
-          } else if (estimation > higherBound) {
-            estimation = higherBound;
-          }
-        } else {
-          estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
-        }
-        aggregateData.setNumDVs(estimation);
-      }
-      columnStatisticsData.setLongStats(aggregateData);
-    } else {
-      // we need extrapolation
-      LOG.debug("start extrapolation for " + colName);
-
-      Map<String, Integer> indexMap = new HashMap<String, Integer>();
-      for (int index = 0; index < partNames.size(); index++) {
-        indexMap.put(partNames.get(index), index);
-      }
-      Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
-      Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
-      // while we scan the css, we also get the densityAvg, lowerbound and
-      // higerbound when useDensityFunctionForNDVEstimation is true.
-      double densityAvgSum = 0.0;
-      if (ndvEstimator == null) {
-        // if not every partition uses bitvector for ndv, we just fall back to
-        // the traditional extrapolation methods.
-        for (ColumnStatistics cs : css) {
-          String partName = cs.getStatsDesc().getPartName();
-          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-          LongColumnStatsData newData = cso.getStatsData().getLongStats();
-          if (useDensityFunctionForNDVEstimation) {
-            densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
-          }
-          adjustedIndexMap.put(partName, (double) indexMap.get(partName));
-          adjustedStatsMap.put(partName, cso.getStatsData());
-        }
-      } else {
-        // we first merge all the adjacent bitvectors that we could merge and
-        // derive new partition names and index.
-        StringBuilder pseudoPartName = new StringBuilder();
-        double pseudoIndexSum = 0;
-        int length = 0;
-        int curIndex = -1;
-        LongColumnStatsData aggregateData = null;
-        for (ColumnStatistics cs : css) {
-          String partName = cs.getStatsDesc().getPartName();
-          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-          LongColumnStatsData newData = cso.getStatsData().getLongStats();
-          // newData.isSetBitVectors() should be true for sure because we
-          // already checked it before.
-          if (indexMap.get(partName) != curIndex) {
-            // There is bitvector, but it is not adjacent to the previous ones.
-            if (length > 0) {
-              // we have to set ndv
-              adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
-              aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
-              ColumnStatisticsData csd = new ColumnStatisticsData();
-              csd.setLongStats(aggregateData);
-              adjustedStatsMap.put(pseudoPartName.toString(), csd);
-              if (useDensityFunctionForNDVEstimation) {
-                densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
-              }
-              // reset everything
-              pseudoPartName = new StringBuilder();
-              pseudoIndexSum = 0;
-              length = 0;
-              ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
-            }
-            aggregateData = null;
-          }
-          curIndex = indexMap.get(partName);
-          pseudoPartName.append(partName);
-          pseudoIndexSum += curIndex;
-          length++;
-          curIndex++;
-          if (aggregateData == null) {
-            aggregateData = newData.deepCopy();
-          } else {
-            aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
-            aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
-                newData.getHighValue()));
-            aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-          }
-          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
-              .getNumDistinctValueEstimator(newData.getBitVectors()));
-        }
-        if (length > 0) {
-          // we have to set ndv
-          adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
-          aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
-          ColumnStatisticsData csd = new ColumnStatisticsData();
-          csd.setLongStats(aggregateData);
-          adjustedStatsMap.put(pseudoPartName.toString(), csd);
-          if (useDensityFunctionForNDVEstimation) {
-            densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
-          }
-        }
-      }
-      extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
-          adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
-    }
-    statsObj.setStatsData(columnStatisticsData);
-    LOG.debug("Ndv estimatation for " + colName + " is "
-        + columnStatisticsData.getLongStats().getNumDVs());
-    return statsObj;
-  }
-
-  @Override
-  public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
-      int numPartsWithStats, Map<String, Double> adjustedIndexMap,
-      Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
-    int rightBorderInd = numParts;
-    LongColumnStatsData extrapolateLongData = new LongColumnStatsData();
-    Map<String, LongColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
-    for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
-      extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getLongStats());
-    }
-    List<Map.Entry<String, LongColumnStatsData>> list = new LinkedList<Map.Entry<String, LongColumnStatsData>>(
-        extractedAdjustedStatsMap.entrySet());
-    // get the lowValue
-    Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
-      public int compare(Map.Entry<String, LongColumnStatsData> o1,
-          Map.Entry<String, LongColumnStatsData> o2) {
-        return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
-      }
-    });
-    double minInd = adjustedIndexMap.get(list.get(0).getKey());
-    double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
-    long lowValue = 0;
-    long min = list.get(0).getValue().getLowValue();
-    long max = list.get(list.size() - 1).getValue().getLowValue();
-    if (minInd == maxInd) {
-      lowValue = min;
-    } else if (minInd < maxInd) {
-      // left border is the min
-      lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
-    } else {
-      // right border is the min
-      lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
-    }
-
-    // get the highValue
-    Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
-      public int compare(Map.Entry<String, LongColumnStatsData> o1,
-          Map.Entry<String, LongColumnStatsData> o2) {
-        return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
-      }
-    });
-    minInd = adjustedIndexMap.get(list.get(0).getKey());
-    maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
-    long highValue = 0;
-    min = list.get(0).getValue().getHighValue();
-    max = list.get(list.size() - 1).getValue().getHighValue();
-    if (minInd == maxInd) {
-      highValue = min;
-    } else if (minInd < maxInd) {
-      // right border is the max
-      highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
-    } else {
-      // left border is the max
-      highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
-    }
-
-    // get the #nulls
-    long numNulls = 0;
-    for (Map.Entry<String, LongColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
-      numNulls += entry.getValue().getNumNulls();
-    }
-    // we scale up sumNulls based on the number of partitions
-    numNulls = numNulls * numParts / numPartsWithStats;
-
-    // get the ndv
-    long ndv = 0;
-    Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
-      public int compare(Map.Entry<String, LongColumnStatsData> o1,
-          Map.Entry<String, LongColumnStatsData> o2) {
-        return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
-      }
-    });
-    long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
-    long higherBound = 0;
-    for (Map.Entry<String, LongColumnStatsData> entry : list) {
-      higherBound += entry.getValue().getNumDVs();
-    }
-    if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
-      ndv = (long) ((highValue - lowValue) / densityAvg);
-      if (ndv < lowerBound) {
-        ndv = lowerBound;
-      } else if (ndv > higherBound) {
-        ndv = higherBound;
-      }
-    } else {
-      minInd = adjustedIndexMap.get(list.get(0).getKey());
-      maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
-      min = list.get(0).getValue().getNumDVs();
-      max = list.get(list.size() - 1).getValue().getNumDVs();
-      if (minInd == maxInd) {
-        ndv = min;
-      } else if (minInd < maxInd) {
-        // right border is the max
-        ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
-      } else {
-        // left border is the max
-        ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
-      }
-    }
-    extrapolateLongData.setLowValue(lowValue);
-    extrapolateLongData.setHighValue(highValue);
-    extrapolateLongData.setNumNulls(numNulls);
-    extrapolateLongData.setNumDVs(ndv);
-    extrapolateData.setLongStats(extrapolateLongData);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java
deleted file mode 100644
index 4c2d1bc..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-
-public class BinaryColumnStatsMerger extends ColumnStatsMerger {
-
-  @Override
-  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
-    BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats();
-    BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats();
-    aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
-    aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
-    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java
deleted file mode 100644
index 8e50153..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-
-public class BooleanColumnStatsMerger extends ColumnStatsMerger {
-
-  @Override
-  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
-    BooleanColumnStatsData aggregateData = aggregateColStats.getStatsData().getBooleanStats();
-    BooleanColumnStatsData newData = newColStats.getStatsData().getBooleanStats();
-    aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
-    aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
-    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java
deleted file mode 100644
index 474d4dd..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class ColumnStatsMerger {
-  protected final Logger LOG = LoggerFactory.getLogger(ColumnStatsMerger.class.getName());
-
-  public abstract void merge(ColumnStatisticsObj aggregateColStats,
-      ColumnStatisticsObj newColStats);
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java
deleted file mode 100644
index 0ce1847..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-
-public class ColumnStatsMergerFactory {
-
-  private ColumnStatsMergerFactory() {
-  }
-
-  public static ColumnStatsMerger getColumnStatsMerger(ColumnStatisticsObj statsObjNew,
-      ColumnStatisticsObj statsObjOld) {
-    ColumnStatsMerger agg;
-    _Fields typeNew = statsObjNew.getStatsData().getSetField();
-    _Fields typeOld = statsObjOld.getStatsData().getSetField();
-    // make sure that they have the same type
-    typeNew = typeNew == typeOld ? typeNew : null;
-    switch (typeNew) {
-    case BOOLEAN_STATS:
-      agg = new BooleanColumnStatsMerger();
-      break;
-    case LONG_STATS: {
-      agg = new LongColumnStatsMerger();
-      break;
-    }
-    case DOUBLE_STATS: {
-      agg = new DoubleColumnStatsMerger();
-      break;
-    }
-    case STRING_STATS: {
-      agg = new StringColumnStatsMerger();
-      break;
-    }
-    case BINARY_STATS:
-      agg = new BinaryColumnStatsMerger();
-      break;
-    case DECIMAL_STATS: {
-      agg = new DecimalColumnStatsMerger();
-      break;
-    }
-    case DATE_STATS: {
-      agg = new DateColumnStatsMerger();
-      break;
-    }
-    default:
-      throw new IllegalArgumentException("Unknown stats type " + typeNew.toString());
-    }
-    return agg;
-  }
-
-  public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
-    ColumnStatisticsObj cso = new ColumnStatisticsObj();
-    ColumnStatisticsData csd = new ColumnStatisticsData();
-    cso.setColName(colName);
-    cso.setColType(colType);
-    switch (type) {
-    case BOOLEAN_STATS:
-      csd.setBooleanStats(new BooleanColumnStatsData());
-      break;
-
-    case LONG_STATS:
-      csd.setLongStats(new LongColumnStatsData());
-      break;
-
-    case DOUBLE_STATS:
-      csd.setDoubleStats(new DoubleColumnStatsData());
-      break;
-
-    case STRING_STATS:
-      csd.setStringStats(new StringColumnStatsData());
-      break;
-
-    case BINARY_STATS:
-      csd.setBinaryStats(new BinaryColumnStatsData());
-      break;
-
-    case DECIMAL_STATS:
-      csd.setDecimalStats(new DecimalColumnStatsData());
-      break;
-
-    case DATE_STATS:
-      csd.setDateStats(new DateColumnStatsData());
-      break;
-
-    default:
-      throw new IllegalArgumentException("Unknown stats type");
-    }
-
-    cso.setStatsData(csd);
-    return cso;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java
deleted file mode 100644
index 2542a00..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Date;
-import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
-
-public class DateColumnStatsMerger extends ColumnStatsMerger {
-  @Override
-  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
-    DateColumnStatsData aggregateData = aggregateColStats.getStatsData().getDateStats();
-    DateColumnStatsData newData = newColStats.getStatsData().getDateStats();
-    Date lowValue = aggregateData.getLowValue().compareTo(newData.getLowValue()) < 0 ? aggregateData
-        .getLowValue() : newData.getLowValue();
-    aggregateData.setLowValue(lowValue);
-    Date highValue = aggregateData.getHighValue().compareTo(newData.getHighValue()) >= 0 ? aggregateData
-        .getHighValue() : newData.getHighValue();
-    aggregateData.setHighValue(highValue);
-    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
-        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
-      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
-    } else {
-      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
-      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(newData.getBitVectors());
-      long ndv = -1;
-      if (oldEst.canMerge(newEst)) {
-        oldEst.mergeEstimators(newEst);
-        ndv = oldEst.estimateNumDistinctValues();
-        aggregateData.setBitVectors(oldEst.serialize());
-      } else {
-        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
-      }
-      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
-          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
-      aggregateData.setNumDVs(ndv);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java
deleted file mode 100644
index 4e8e129..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-
-public class DecimalColumnStatsMerger extends ColumnStatsMerger {
-  @Override
-  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
-    DecimalColumnStatsData aggregateData = aggregateColStats.getStatsData().getDecimalStats();
-    DecimalColumnStatsData newData = newColStats.getStatsData().getDecimalStats();
-    Decimal lowValue = aggregateData.getLowValue() != null
-        && (aggregateData.getLowValue().compareTo(newData.getLowValue()) > 0) ? aggregateData
-        .getLowValue() : newData.getLowValue();
-    aggregateData.setLowValue(lowValue);
-    Decimal highValue = aggregateData.getHighValue() != null
-        && (aggregateData.getHighValue().compareTo(newData.getHighValue()) > 0) ? aggregateData
-        .getHighValue() : newData.getHighValue();
-    aggregateData.setHighValue(highValue);
-    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
-        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
-      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
-    } else {
-      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
-      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(newData.getBitVectors());
-      long ndv = -1;
-      if (oldEst.canMerge(newEst)) {
-        oldEst.mergeEstimators(newEst);
-        ndv = oldEst.estimateNumDistinctValues();
-        aggregateData.setBitVectors(oldEst.serialize());
-      } else {
-        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
-      }
-      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
-          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
-      aggregateData.setNumDVs(ndv);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java
deleted file mode 100644
index 4ef5c39..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-
-public class DoubleColumnStatsMerger extends ColumnStatsMerger {
-  @Override
-  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
-    DoubleColumnStatsData aggregateData = aggregateColStats.getStatsData().getDoubleStats();
-    DoubleColumnStatsData newData = newColStats.getStatsData().getDoubleStats();
-    aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
-    aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
-    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
-        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
-      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
-    } else {
-      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
-      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(newData.getBitVectors());
-      long ndv = -1;
-      if (oldEst.canMerge(newEst)) {
-        oldEst.mergeEstimators(newEst);
-        ndv = oldEst.estimateNumDistinctValues();
-        aggregateData.setBitVectors(oldEst.serialize());
-      } else {
-        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
-      }
-      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
-          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
-      aggregateData.setNumDVs(ndv);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java
deleted file mode 100644
index acf7f03..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-
-public class LongColumnStatsMerger extends ColumnStatsMerger {
-  @Override
-  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
-    LongColumnStatsData aggregateData = aggregateColStats.getStatsData().getLongStats();
-    LongColumnStatsData newData = newColStats.getStatsData().getLongStats();
-    aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
-    aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
-    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
-        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
-      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
-    } else {
-      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
-      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(newData.getBitVectors());
-      long ndv = -1;
-      if (oldEst.canMerge(newEst)) {
-        oldEst.mergeEstimators(newEst);
-        ndv = oldEst.estimateNumDistinctValues();
-        aggregateData.setBitVectors(oldEst.serialize());
-      } else {
-        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
-      }
-      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
-          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
-      aggregateData.setNumDVs(ndv);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java
deleted file mode 100644
index b3cd33c..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-
-public class StringColumnStatsMerger extends ColumnStatsMerger {
-  @Override
-  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
-    StringColumnStatsData aggregateData = aggregateColStats.getStatsData().getStringStats();
-    StringColumnStatsData newData = newColStats.getStatsData().getStringStats();
-    aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
-    aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
-    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-    if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
-        || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
-      aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
-    } else {
-      NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(aggregateData.getBitVectors());
-      NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
-          .getNumDistinctValueEstimator(newData.getBitVectors());
-      long ndv = -1;
-      if (oldEst.canMerge(newEst)) {
-        oldEst.mergeEstimators(newEst);
-        ndv = oldEst.estimateNumDistinctValues();
-        aggregateData.setBitVectors(oldEst.serialize());
-      } else {
-        ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
-      }
-      LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
-          + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
-      aggregateData.setNumDVs(ndv);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
----------------------------------------------------------------------
diff --git a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto b/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
deleted file mode 100644
index 53c381b..0000000
--- a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
+++ /dev/null
@@ -1,368 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-enum PrincipalType {
-    USER = 0;
-    ROLE = 1;
-}
-
-message AggrStats {
-  required int64 parts_found = 1;
-  repeated ColumnStats col_stats = 2;
-}
-
-message AggrStatsBloomFilter {
-  message BloomFilter {
-    required int32 num_bits = 1;
-    required int32 num_funcs = 2;
-    repeated int64 bits = 3;
-  }
-  required bytes db_name = 1;
-  required bytes table_name = 2;
-  required BloomFilter bloom_filter = 3;
-  required int64 aggregated_at = 4;
-}
-
-message AggrStatsInvalidatorFilter {
-  message Entry {
-    required bytes db_name = 1;
-    required bytes table_name = 2;
-    required bytes part_name = 3;
-  }
-
-  repeated Entry to_invalidate = 1;
-  required int64 run_every = 2;
-  required int64 max_cache_entry_life = 3;
-}
-
-message ColumnStats {
-
-  message BooleanStats {
-    optional int64 num_trues = 1;
-    optional int64 num_falses = 2;
-  }
-
-  message LongStats {
-    optional sint64 low_value = 1;
-    optional sint64 high_value = 2;
-  }
-
-  message DoubleStats {
-    optional double low_value = 1;
-    optional double high_value = 2;
-  }
-
-  message StringStats {
-    optional int64 max_col_length = 1;
-    optional double avg_col_length = 2;
-  }
-
-  message DecimalStats {
-    message Decimal {
-      required bytes unscaled = 1;
-      required int32 scale = 2;
-    }
-    optional Decimal low_value = 1;
-    optional Decimal high_value = 2;
-  }
-
-  optional int64 last_analyzed = 1;
-  required string column_type = 2;
-  optional int64 num_nulls = 3;
-  optional int64 num_distinct_values = 4;
-  optional BooleanStats bool_stats = 5;
-  optional LongStats long_stats = 6;
-  optional DoubleStats double_stats = 7;
-  optional StringStats string_stats = 8;
-  optional StringStats binary_stats = 9;
-  optional DecimalStats decimal_stats = 10;
-  optional string column_name = 11;
-  optional string bit_vectors = 12;
-}
-
-message Database {
-  optional string description = 1;
-  optional string uri = 2;
-  optional Parameters parameters = 3;
-  optional PrincipalPrivilegeSet privileges = 4;
-  optional string owner_name = 5;
-  optional PrincipalType owner_type = 6;
-}
-
-message DelegationToken {
-  required string token_str = 1;
-}
-
-message FieldSchema {
-  required string name = 1;
-  required string type = 2;
-  optional string comment = 3;
-}
-
-message Function {
-  enum FunctionType {
-    JAVA = 1;
-  }
-
-  message ResourceUri {
-    enum ResourceType {
-      JAR = 1;
-      FILE = 2;
-      ARCHIVE = 3;
-    }
-    required ResourceType resource_type = 1;
-    required string uri = 2;
-  }
-
-  optional string class_name = 1;
-  optional string owner_name = 2;
-  optional PrincipalType owner_type = 3;
-  optional sint64 create_time = 4;
-  optional FunctionType function_type = 5;
-  repeated ResourceUri resource_uris = 6;
-}
-
-message MasterKey {
-  required string master_key = 1;
-}
-
-message ParameterEntry {
-  required string key = 1;
-  required string value = 2;
-}
-
-message Parameters {
-  repeated ParameterEntry parameter = 1;
-}
-
-message Partition {
-  optional int64 create_time = 1;
-  optional int64 last_access_time = 2;
-  optional string location = 3;
-  optional Parameters sd_parameters = 4; // storage descriptor parameters
-  required bytes sd_hash = 5;
-  optional Parameters parameters = 6; // partition parameters
-  // We don't support partition level privileges
-}
-
-message PrincipalPrivilegeSetEntry {
-  required string principal_name = 1;
-  repeated PrivilegeGrantInfo privileges = 2;
-}
-
-message PrincipalPrivilegeSet {
-  repeated PrincipalPrivilegeSetEntry users = 1;
-  repeated PrincipalPrivilegeSetEntry roles = 2;
-}
-
-message PrivilegeGrantInfo {
-  optional string privilege = 1;
-  optional int64 create_time = 2;
-  optional string grantor = 3;
-  optional PrincipalType grantor_type = 4;
-  optional bool grant_option = 5;
-}
-
-message RoleGrantInfo {
-  required string principal_name = 1;
-  required PrincipalType principal_type = 2;
-  optional int64 add_time = 3;
-  optional string grantor = 4;
-  optional PrincipalType grantor_type = 5;
-  optional bool grant_option = 6;
-}
-
-message RoleGrantInfoList {
-  repeated RoleGrantInfo grant_info = 1;
-}
-
-message RoleList {
-  repeated string role = 1;
-}
-
-message Role {
-  optional int64 create_time = 1;
-  optional string owner_name = 2;
-}
-
-message StorageDescriptor {
-  message Order {
-    required string column_name = 1;
-    optional sint32 order = 2 [default = 1];
-  }
-
-  message SerDeInfo {
-    optional string name = 1;
-    optional string serialization_lib = 2;
-    optional Parameters parameters = 3;
-  }
-
-  message SkewedInfo {
-    message SkewedColValueList {
-      repeated string skewed_col_value = 1;
-    }
-  
-    message SkewedColValueLocationMap {
-      repeated string key = 1;
-      required string value = 2;
-    }
-
-    repeated string skewed_col_names = 1;
-    repeated SkewedColValueList skewed_col_values = 2;
-    repeated SkewedColValueLocationMap skewed_col_value_location_maps = 3;
-  }
-
-  repeated FieldSchema cols = 1;
-  optional string input_format = 2;
-  optional string output_format = 3;
-  optional bool is_compressed = 4;
-  optional sint32 num_buckets = 5;
-  optional SerDeInfo serde_info = 6;
-  repeated string bucket_cols = 7;
-  repeated Order sort_cols = 8;
-  optional SkewedInfo skewed_info = 9;
-  optional bool stored_as_sub_directories = 10;
-}
-
-message Table {
-  optional string owner = 1;
-  optional int64 create_time = 2;
-  optional int64 last_access_time = 3;
-  optional int64 retention = 4;
-  optional string location = 5;
-  optional Parameters sd_parameters = 6; // storage descriptor parameters
-  required bytes sd_hash = 7;
-  repeated FieldSchema partition_keys = 8;
-  optional Parameters parameters = 9;
-  optional string view_original_text = 10;
-  optional string view_expanded_text = 11;
-  optional string table_type = 12;
-  optional PrincipalPrivilegeSet privileges = 13;
-  optional bool is_temporary = 14;
-  optional bool is_rewrite_enabled = 15;
-}
-
-message Index {
-  optional string indexHandlerClass = 1; // reserved
-  required string dbName = 2;
-  required string origTableName = 3;
-  optional string location = 4;
-  optional Parameters sd_parameters = 5; // storage descriptor parameters
-  optional int32 createTime = 6;
-  optional int32 lastAccessTime = 7;
-  optional string indexTableName = 8;
-  optional bytes sd_hash = 9;
-  optional Parameters parameters = 10;
-  optional bool deferredRebuild = 11;
-}
-
-message PartitionKeyComparator {
-  required string names = 1;
-  required string types = 2;
-  message Mark {
-    required string value = 1;
-    required bool inclusive = 2;
-  }
-  message Range {
-    required string key = 1;
-    optional Mark start = 2;
-    optional Mark end = 3;
-  }
-  message Operator {
-    enum Type {
-      LIKE = 0;
-      NOTEQUALS = 1;
-    }
-    required Type type = 1;
-    required string key = 2;
-    required string val = 3;
-  }
-  repeated Operator op = 3;
-  repeated Range range = 4;
-}
-
-message PrimaryKey {
-  message PrimaryKeyColumn {
-    required string column_name = 1;
-    required sint32 key_seq = 2;
-  }
-
-  required string pk_name = 1;
-  repeated PrimaryKeyColumn cols = 2;
-  optional bool enable_constraint = 3;
-  optional bool validate_constraint = 4;
-  optional bool rely_constraint = 5;
-}
-
-message ForeignKeys {
-  message ForeignKey {
-    message ForeignKeyColumn {
-      required string column_name = 1;
-      required string referenced_column_name = 2;
-      required sint32 key_seq = 3;
-    }
-
-    required string fk_name = 1;
-    required string referenced_db_name = 2;
-    required string referenced_table_name = 3;
-    optional string referenced_pk_name = 4;
-    optional int32 update_rule = 5;
-    optional int32 delete_rule = 6;
-    repeated ForeignKeyColumn cols = 7;
-    optional bool enable_constraint = 8;
-    optional bool validate_constraint = 9;
-    optional bool rely_constraint = 10;
-  }
-
-  repeated ForeignKey fks = 1;
-}
-
-message UniqueConstraints {
-  message UniqueConstraint {
-    message UniqueConstraintColumn {
-      required string column_name = 1;
-      required sint32 key_seq = 2;
-    }
-
-    required string uk_name = 1;
-    repeated UniqueConstraintColumn cols = 2;
-    optional bool enable_constraint = 3;
-    optional bool validate_constraint = 4;
-    optional bool rely_constraint = 5;
-  }
-
-  repeated UniqueConstraint uks = 1;
-}
-
-message NotNullConstraints {
-  message NotNullConstraint {
-    message NotNullConstraintColumn {
-      required string column_name = 1;
-    }
-
-    required string nn_name = 1;
-    repeated NotNullConstraintColumn cols = 2;
-    optional bool enable_constraint = 3;
-    optional bool validate_constraint = 4;
-    optional bool rely_constraint = 5;
-  }
-
-  repeated NotNullConstraint nns = 1;
-}
-

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
deleted file mode 100644
index 784648a..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.FileFormatProxy;
-import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-
-/**
- * Mock utilities for HBaseStore testing
- */
-public class MockUtils {
-
-  /**
-   * The default impl is in ql package and is not available in unit tests.
-   */
-  public static class NOOPProxy implements PartitionExpressionProxy {
-
-    @Override
-    public String convertExprToFilter(byte[] expr) throws MetaException {
-      return null;
-    }
-
-    @Override
-    public boolean filterPartitionsByExpr(List<String> partColumnNames,
-        List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr, String defaultPartitionName,
-        List<String> partitionNames) throws MetaException {
-      return false;
-    }
-
-    @Override
-    public SearchArgument createSarg(byte[] expr) {
-      return null;
-    }
-
-    @Override
-    public FileMetadataExprType getMetadataType(String inputFormat) {
-      return null;
-    }
-
-    @Override
-    public FileFormatProxy getFileFormatProxy(FileMetadataExprType type) {
-      return null;
-    }
-  }
-
-  static HBaseStore init(Configuration conf, HTableInterface htable,
-                         final SortedMap<String, Cell> rows) throws IOException {
-    ((HiveConf)conf).setVar(ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, NOOPProxy.class.getName());
-    Mockito.when(htable.get(Mockito.any(Get.class))).thenAnswer(new Answer<Result>() {
-      @Override
-      public Result answer(InvocationOnMock invocation) throws Throwable {
-        Get get = (Get) invocation.getArguments()[0];
-        Cell cell = rows.get(new String(get.getRow()));
-        if (cell == null) {
-          return new Result();
-        } else {
-          return Result.create(new Cell[]{cell});
-        }
-      }
-    });
-
-    Mockito.when(htable.get(Mockito.anyListOf(Get.class))).thenAnswer(new Answer<Result[]>() {
-      @Override
-      public Result[] answer(InvocationOnMock invocation) throws Throwable {
-        @SuppressWarnings("unchecked")
-        List<Get> gets = (List<Get>) invocation.getArguments()[0];
-        Result[] results = new Result[gets.size()];
-        for (int i = 0; i < gets.size(); i++) {
-          Cell cell = rows.get(new String(gets.get(i).getRow()));
-          Result result;
-          if (cell == null) {
-            result = new Result();
-          } else {
-            result = Result.create(new Cell[]{cell});
-          }
-          results[i] = result;
-        }
-        return results;
-      }
-    });
-
-    Mockito.when(htable.getScanner(Mockito.any(Scan.class))).thenAnswer(new Answer<ResultScanner>() {
-      @Override
-      public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
-        Scan scan = (Scan)invocation.getArguments()[0];
-        List<Result> results = new ArrayList<Result>();
-        String start = new String(scan.getStartRow());
-        String stop = new String(scan.getStopRow());
-        SortedMap<String, Cell> sub = rows.subMap(start, stop);
-        for (Map.Entry<String, Cell> e : sub.entrySet()) {
-          results.add(Result.create(new Cell[]{e.getValue()}));
-        }
-
-        final Iterator<Result> iter = results.iterator();
-
-        return new ResultScanner() {
-          @Override
-          public Result next() throws IOException {
-            return null;
-          }
-
-          @Override
-          public Result[] next(int nbRows) throws IOException {
-            return new Result[0];
-          }
-
-          @Override
-          public void close() {
-
-          }
-
-          @Override
-          public Iterator<Result> iterator() {
-            return iter;
-          }
-        };
-      }
-    });
-
-    Mockito.doAnswer(new Answer<Void>() {
-      @Override
-      public Void answer(InvocationOnMock invocation) throws Throwable {
-        Put put = (Put)invocation.getArguments()[0];
-        rows.put(new String(put.getRow()), put.getFamilyCellMap().firstEntry().getValue().get(0));
-        return null;
-      }
-    }).when(htable).put(Mockito.any(Put.class));
-
-    Mockito.when(htable.checkAndPut(Mockito.any(byte[].class), Mockito.any(byte[].class),
-        Mockito.any(byte[].class), Mockito.any(byte[].class), Mockito.any(Put.class))).thenAnswer(
-        new Answer<Boolean>() {
-
-          @Override
-          public Boolean answer(InvocationOnMock invocation) throws Throwable {
-            // Always say it succeeded and overwrite
-            Put put = (Put)invocation.getArguments()[4];
-            rows.put(new String(put.getRow()),
-                put.getFamilyCellMap().firstEntry().getValue().get(0));
-            return true;
-          }
-        });
-
-    Mockito.doAnswer(new Answer<Void>() {
-      @Override
-      public Void answer(InvocationOnMock invocation) throws Throwable {
-        Delete del = (Delete)invocation.getArguments()[0];
-        rows.remove(new String(del.getRow()));
-        return null;
-      }
-    }).when(htable).delete(Mockito.any(Delete.class));
-
-    Mockito.when(htable.checkAndDelete(Mockito.any(byte[].class), Mockito.any(byte[].class),
-        Mockito.any(byte[].class), Mockito.any(byte[].class), Mockito.any(Delete.class))).thenAnswer(
-        new Answer<Boolean>() {
-
-          @Override
-          public Boolean answer(InvocationOnMock invocation) throws Throwable {
-            // Always say it succeeded
-            Delete del = (Delete)invocation.getArguments()[4];
-            rows.remove(new String(del.getRow()));
-            return true;
-          }
-        });
-
-    // Mock connection
-    HBaseConnection hconn = Mockito.mock(HBaseConnection.class);
-    Mockito.when(hconn.getHBaseTable(Mockito.anyString())).thenReturn(htable);
-    HiveConf.setVar(conf, HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS, HBaseReadWrite.TEST_CONN);
-    HBaseReadWrite.setTestConnection(hconn);
-    HBaseReadWrite.setConf(conf);
-    HBaseStore store = new HBaseStore();
-    store.setConf(conf);
-    return store;
-  }
-}


[12/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index edfbf3a..48602bd 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -22,11 +22,16 @@ import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -37,6 +42,10 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -46,6 +55,9 @@ import com.google.common.collect.Maps;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.ListUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.metastore.api.Decimal;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SkewedInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -114,6 +126,7 @@ public class MetaStoreUtils {
   // configuration parameter documentation
   // HIVE_SUPPORT_SPECICAL_CHARACTERS_IN_TABLE_NAMES in HiveConf as well.
   public static final char[] specialCharactersInTableNames = new char[] { '/' };
+  final static Charset ENCODING = StandardCharsets.UTF_8;
 
   public static Table createColumnsetSchema(String name, List<String> columns,
       List<String> partCols, Configuration conf) throws MetaException {
@@ -1988,4 +2001,88 @@ public class MetaStoreUtils {
     return colStats;
   }
 
+  /**
+   * Produce a hash for the storage descriptor
+   * @param sd storage descriptor to hash
+   * @param md message descriptor to use to generate the hash
+   * @return the hash as a byte array
+   */
+  public static byte[] hashStorageDescriptor(StorageDescriptor sd, MessageDigest md)  {
+    // Note all maps and lists have to be absolutely sorted.  Otherwise we'll produce different
+    // results for hashes based on the OS or JVM being used.
+    md.reset();
+    for (FieldSchema fs : sd.getCols()) {
+      md.update(fs.getName().getBytes(ENCODING));
+      md.update(fs.getType().getBytes(ENCODING));
+      if (fs.getComment() != null) md.update(fs.getComment().getBytes(ENCODING));
+    }
+    if (sd.getInputFormat() != null) {
+      md.update(sd.getInputFormat().getBytes(ENCODING));
+    }
+    if (sd.getOutputFormat() != null) {
+      md.update(sd.getOutputFormat().getBytes(ENCODING));
+    }
+    md.update(sd.isCompressed() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
+    md.update(Integer.toString(sd.getNumBuckets()).getBytes(ENCODING));
+    if (sd.getSerdeInfo() != null) {
+      SerDeInfo serde = sd.getSerdeInfo();
+      if (serde.getName() != null) {
+        md.update(serde.getName().getBytes(ENCODING));
+      }
+      if (serde.getSerializationLib() != null) {
+        md.update(serde.getSerializationLib().getBytes(ENCODING));
+      }
+      if (serde.getParameters() != null) {
+        SortedMap<String, String> params = new TreeMap<>(serde.getParameters());
+        for (Entry<String, String> param : params.entrySet()) {
+          md.update(param.getKey().getBytes(ENCODING));
+          md.update(param.getValue().getBytes(ENCODING));
+        }
+      }
+    }
+    if (sd.getBucketCols() != null) {
+      List<String> bucketCols = new ArrayList<>(sd.getBucketCols());
+      for (String bucket : bucketCols) md.update(bucket.getBytes(ENCODING));
+    }
+    if (sd.getSortCols() != null) {
+      SortedSet<Order> orders = new TreeSet<>(sd.getSortCols());
+      for (Order order : orders) {
+        md.update(order.getCol().getBytes(ENCODING));
+        md.update(Integer.toString(order.getOrder()).getBytes(ENCODING));
+      }
+    }
+    if (sd.getSkewedInfo() != null) {
+      SkewedInfo skewed = sd.getSkewedInfo();
+      if (skewed.getSkewedColNames() != null) {
+        SortedSet<String> colnames = new TreeSet<>(skewed.getSkewedColNames());
+        for (String colname : colnames) md.update(colname.getBytes(ENCODING));
+      }
+      if (skewed.getSkewedColValues() != null) {
+        SortedSet<String> sortedOuterList = new TreeSet<>();
+        for (List<String> innerList : skewed.getSkewedColValues()) {
+          SortedSet<String> sortedInnerList = new TreeSet<>(innerList);
+          sortedOuterList.add(StringUtils.join(sortedInnerList, "."));
+        }
+        for (String colval : sortedOuterList) md.update(colval.getBytes(ENCODING));
+      }
+      if (skewed.getSkewedColValueLocationMaps() != null) {
+        SortedMap<String, String> sortedMap = new TreeMap<>();
+        for (Entry<List<String>, String> smap : skewed.getSkewedColValueLocationMaps().entrySet()) {
+          SortedSet<String> sortedKey = new TreeSet<>(smap.getKey());
+          sortedMap.put(StringUtils.join(sortedKey, "."), smap.getValue());
+        }
+        for (Entry<String, String> e : sortedMap.entrySet()) {
+          md.update(e.getKey().getBytes(ENCODING));
+          md.update(e.getValue().getBytes(ENCODING));
+        }
+      }
+      md.update(sd.isStoredAsSubDirectories() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
+    }
+
+    return md.digest();
+  }
+
+  public static double decimalToDouble(Decimal decimal) {
+    return new BigDecimal(new BigInteger(decimal.getUnscaled()), decimal.getScale()).doubleValue();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
index b6fb4fd..80b17e0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.StatObjectConverter;
 import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
@@ -40,7 +41,6 @@ import org.apache.hadoop.hive.metastore.api.TableMeta;
 import org.apache.hadoop.hive.metastore.cache.CachedStore.PartitionWrapper;
 import org.apache.hadoop.hive.metastore.cache.CachedStore.StorageDescriptorWrapper;
 import org.apache.hadoop.hive.metastore.cache.CachedStore.TableWrapper;
-import org.apache.hadoop.hive.metastore.hbase.HBaseUtils;
 import org.apache.hive.common.util.HiveStringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -117,7 +117,7 @@ public class SharedCache {
     }
     TableWrapper wrapper;
     if (tbl.getSd() != null) {
-      byte[] sdHash = HBaseUtils.hashStorageDescriptor(tbl.getSd(), md);
+      byte[] sdHash = MetaStoreUtils.hashStorageDescriptor(tbl.getSd(), md);
       StorageDescriptor sd = tbl.getSd();
       increSd(sd, sdHash);
       tblCopy.setSd(null);
@@ -285,7 +285,7 @@ public class SharedCache {
     Partition partCopy = part.deepCopy();
     PartitionWrapper wrapper;
     if (part.getSd()!=null) {
-      byte[] sdHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
+      byte[] sdHash = MetaStoreUtils.hashStorageDescriptor(part.getSd(), md);
       StorageDescriptor sd = part.getSd();
       increSd(sd, sdHash);
       partCopy.setSd(null);

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
new file mode 100644
index 0000000..e6c836b
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+public class BinaryColumnStatsAggregator extends ColumnStatsAggregator {
+
+  @Override
+  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+      List<ColumnStatistics> css) throws MetaException {
+    ColumnStatisticsObj statsObj = null;
+    BinaryColumnStatsData aggregateData = null;
+    String colType = null;
+    for (ColumnStatistics cs : css) {
+      if (cs.getStatsObjSize() != 1) {
+        throw new MetaException(
+            "The number of columns should be exactly one in aggrStats, but found "
+                + cs.getStatsObjSize());
+      }
+      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+      if (statsObj == null) {
+        colType = cso.getColType();
+        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+            .getStatsData().getSetField());
+      }
+      BinaryColumnStatsData newData = cso.getStatsData().getBinaryStats();
+      if (aggregateData == null) {
+        aggregateData = newData.deepCopy();
+      } else {
+        aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
+        aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
+        aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+      }
+    }
+    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+    columnStatisticsData.setBinaryStats(aggregateData);
+    statsObj.setStatsData(columnStatisticsData);
+    return statsObj;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
new file mode 100644
index 0000000..a34bc9f
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+public class BooleanColumnStatsAggregator extends ColumnStatsAggregator {
+
+  @Override
+  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+      List<ColumnStatistics> css) throws MetaException {
+    ColumnStatisticsObj statsObj = null;
+    BooleanColumnStatsData aggregateData = null;
+    String colType = null;
+    for (ColumnStatistics cs : css) {
+      if (cs.getStatsObjSize() != 1) {
+        throw new MetaException(
+            "The number of columns should be exactly one in aggrStats, but found "
+                + cs.getStatsObjSize());
+      }
+      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+      if (statsObj == null) {
+        colType = cso.getColType();
+        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+            .getStatsData().getSetField());
+      }
+      BooleanColumnStatsData newData = cso.getStatsData().getBooleanStats();
+      if (aggregateData == null) {
+        aggregateData = newData.deepCopy();
+      } else {
+        aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
+        aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
+        aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+      }
+    }
+    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+    columnStatisticsData.setBooleanStats(aggregateData);
+    statsObj.setStatsData(columnStatisticsData);
+    return statsObj;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
new file mode 100644
index 0000000..a52e5e5
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+public abstract class ColumnStatsAggregator {
+  public boolean useDensityFunctionForNDVEstimation;
+  public double ndvTuner;
+  public abstract ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+      List<ColumnStatistics> css) throws MetaException;
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java
new file mode 100644
index 0000000..173e06f
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+
+public class ColumnStatsAggregatorFactory {
+
+  private ColumnStatsAggregatorFactory() {
+  }
+
+  public static ColumnStatsAggregator getColumnStatsAggregator(_Fields type,
+      boolean useDensityFunctionForNDVEstimation, double ndvTuner) {
+    ColumnStatsAggregator agg;
+    switch (type) {
+    case BOOLEAN_STATS:
+      agg = new BooleanColumnStatsAggregator();
+      break;
+    case LONG_STATS:
+      agg = new LongColumnStatsAggregator();
+      break;
+    case DATE_STATS:
+      agg = new DateColumnStatsAggregator();
+      break;
+    case DOUBLE_STATS:
+      agg = new DoubleColumnStatsAggregator();
+      break;
+    case STRING_STATS:
+      agg = new StringColumnStatsAggregator();
+      break;
+    case BINARY_STATS:
+      agg = new BinaryColumnStatsAggregator();
+      break;
+    case DECIMAL_STATS:
+      agg = new DecimalColumnStatsAggregator();
+      break;
+    default:
+      throw new RuntimeException("Woh, bad.  Unknown stats type " + type.toString());
+    }
+    agg.useDensityFunctionForNDVEstimation = useDensityFunctionForNDVEstimation;
+    agg.ndvTuner = ndvTuner;
+    return agg;
+  }
+
+  public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
+    ColumnStatisticsObj cso = new ColumnStatisticsObj();
+    ColumnStatisticsData csd = new ColumnStatisticsData();
+    cso.setColName(colName);
+    cso.setColType(colType);
+    switch (type) {
+    case BOOLEAN_STATS:
+      csd.setBooleanStats(new BooleanColumnStatsData());
+      break;
+
+    case LONG_STATS:
+      csd.setLongStats(new LongColumnStatsData());
+      break;
+
+    case DATE_STATS:
+      csd.setDateStats(new DateColumnStatsData());
+      break;
+
+    case DOUBLE_STATS:
+      csd.setDoubleStats(new DoubleColumnStatsData());
+      break;
+
+    case STRING_STATS:
+      csd.setStringStats(new StringColumnStatsData());
+      break;
+
+    case BINARY_STATS:
+      csd.setBinaryStats(new BinaryColumnStatsData());
+      break;
+
+    case DECIMAL_STATS:
+      csd.setDecimalStats(new DecimalColumnStatsData());
+      break;
+
+    default:
+      throw new RuntimeException("Woh, bad.  Unknown stats type!");
+    }
+
+    cso.setStatsData(csd);
+    return cso;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java
new file mode 100644
index 0000000..c5e72eb
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java
@@ -0,0 +1,371 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.StatObjectConverter;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implements
+    IExtrapolatePartStatus {
+  
+  private static final Logger LOG = LoggerFactory.getLogger(DecimalColumnStatsAggregator.class);
+
+  @Override
+  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+      List<ColumnStatistics> css) throws MetaException {
+    ColumnStatisticsObj statsObj = null;
+
+    // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+    // bitvectors
+    boolean doAllPartitionContainStats = partNames.size() == css.size();
+    LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
+    NumDistinctValueEstimator ndvEstimator = null;
+    String colType = null;
+    for (ColumnStatistics cs : css) {
+      if (cs.getStatsObjSize() != 1) {
+        throw new MetaException(
+            "The number of columns should be exactly one in aggrStats, but found "
+                + cs.getStatsObjSize());
+      }
+      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+      if (statsObj == null) {
+        colType = cso.getColType();
+        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+            .getStatsData().getSetField());
+      }
+      if (!cso.getStatsData().getDecimalStats().isSetBitVectors()
+          || cso.getStatsData().getDecimalStats().getBitVectors().length() == 0) {
+        ndvEstimator = null;
+        break;
+      } else {
+        // check if all of the bit vectors can merge
+        NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
+            .getNumDistinctValueEstimator(cso.getStatsData().getDecimalStats().getBitVectors());
+        if (ndvEstimator == null) {
+          ndvEstimator = estimator;
+        } else {
+          if (ndvEstimator.canMerge(estimator)) {
+            continue;
+          } else {
+            ndvEstimator = null;
+            break;
+          }
+        }
+      }
+    }
+    if (ndvEstimator != null) {
+      ndvEstimator = NumDistinctValueEstimatorFactory
+          .getEmptyNumDistinctValueEstimator(ndvEstimator);
+    }
+    LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
+    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+    if (doAllPartitionContainStats || css.size() < 2) {
+      DecimalColumnStatsData aggregateData = null;
+      long lowerBound = 0;
+      long higherBound = 0;
+      double densityAvgSum = 0.0;
+      for (ColumnStatistics cs : css) {
+        ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+        DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
+        lowerBound = Math.max(lowerBound, newData.getNumDVs());
+        higherBound += newData.getNumDVs();
+        densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils
+            .decimalToDouble(newData.getLowValue())) / newData.getNumDVs();
+        if (ndvEstimator != null) {
+          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+              .getNumDistinctValueEstimator(newData.getBitVectors()));
+        }
+        if (aggregateData == null) {
+          aggregateData = newData.deepCopy();
+        } else {
+          if (MetaStoreUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils
+              .decimalToDouble(newData.getLowValue())) {
+            aggregateData.setLowValue(aggregateData.getLowValue());
+          } else {
+            aggregateData.setLowValue(newData.getLowValue());
+          }
+          if (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils
+              .decimalToDouble(newData.getHighValue())) {
+            aggregateData.setHighValue(aggregateData.getHighValue());
+          } else {
+            aggregateData.setHighValue(newData.getHighValue());
+          }
+          aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+          aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+        }
+      }
+      if (ndvEstimator != null) {
+        // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
+        // use uniform distribution assumption because we can merge bitvectors
+        // to get a good estimation.
+        aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+      } else {
+        long estimation;
+        if (useDensityFunctionForNDVEstimation) {
+          // We have estimation, lowerbound and higherbound. We use estimation
+          // if it is between lowerbound and higherbound.
+          double densityAvg = densityAvgSum / partNames.size();
+          estimation = (long) ((MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils
+              .decimalToDouble(aggregateData.getLowValue())) / densityAvg);
+          if (estimation < lowerBound) {
+            estimation = lowerBound;
+          } else if (estimation > higherBound) {
+            estimation = higherBound;
+          }
+        } else {
+          estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
+        }
+        aggregateData.setNumDVs(estimation);
+      }
+      columnStatisticsData.setDecimalStats(aggregateData);
+    } else {
+      // we need extrapolation
+      LOG.debug("start extrapolation for " + colName);
+      Map<String, Integer> indexMap = new HashMap<String, Integer>();
+      for (int index = 0; index < partNames.size(); index++) {
+        indexMap.put(partNames.get(index), index);
+      }
+      Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
+      Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
+      // while we scan the css, we also get the densityAvg, lowerbound and
+      // higerbound when useDensityFunctionForNDVEstimation is true.
+      double densityAvgSum = 0.0;
+      if (ndvEstimator == null) {
+        // if not every partition uses bitvector for ndv, we just fall back to
+        // the traditional extrapolation methods.
+        for (ColumnStatistics cs : css) {
+          String partName = cs.getStatsDesc().getPartName();
+          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+          DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
+          if (useDensityFunctionForNDVEstimation) {
+            densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils
+                .decimalToDouble(newData.getLowValue())) / newData.getNumDVs();
+          }
+          adjustedIndexMap.put(partName, (double) indexMap.get(partName));
+          adjustedStatsMap.put(partName, cso.getStatsData());
+        }
+      } else {
+        // we first merge all the adjacent bitvectors that we could merge and
+        // derive new partition names and index.
+        StringBuilder pseudoPartName = new StringBuilder();
+        double pseudoIndexSum = 0;
+        int length = 0;
+        int curIndex = -1;
+        DecimalColumnStatsData aggregateData = null;
+        for (ColumnStatistics cs : css) {
+          String partName = cs.getStatsDesc().getPartName();
+          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+          DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
+          // newData.isSetBitVectors() should be true for sure because we
+          // already checked it before.
+          if (indexMap.get(partName) != curIndex) {
+            // There is bitvector, but it is not adjacent to the previous ones.
+            if (length > 0) {
+              // we have to set ndv
+              adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+              aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+              ColumnStatisticsData csd = new ColumnStatisticsData();
+              csd.setDecimalStats(aggregateData);
+              adjustedStatsMap.put(pseudoPartName.toString(), csd);
+              if (useDensityFunctionForNDVEstimation) {
+                densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils
+                    .decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs();
+              }
+              // reset everything
+              pseudoPartName = new StringBuilder();
+              pseudoIndexSum = 0;
+              length = 0;
+              ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
+            }
+            aggregateData = null;
+          }
+          curIndex = indexMap.get(partName);
+          pseudoPartName.append(partName);
+          pseudoIndexSum += curIndex;
+          length++;
+          curIndex++;
+          if (aggregateData == null) {
+            aggregateData = newData.deepCopy();
+          } else {
+            if (MetaStoreUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils
+                .decimalToDouble(newData.getLowValue())) {
+              aggregateData.setLowValue(aggregateData.getLowValue());
+            } else {
+              aggregateData.setLowValue(newData.getLowValue());
+            }
+            if (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils
+                .decimalToDouble(newData.getHighValue())) {
+              aggregateData.setHighValue(aggregateData.getHighValue());
+            } else {
+              aggregateData.setHighValue(newData.getHighValue());
+            }
+            aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+          }
+          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+              .getNumDistinctValueEstimator(newData.getBitVectors()));
+        }
+        if (length > 0) {
+          // we have to set ndv
+          adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+          aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+          ColumnStatisticsData csd = new ColumnStatisticsData();
+          csd.setDecimalStats(aggregateData);
+          adjustedStatsMap.put(pseudoPartName.toString(), csd);
+          if (useDensityFunctionForNDVEstimation) {
+            densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils
+                .decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs();
+          }
+        }
+      }
+      extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
+          adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
+    }
+    statsObj.setStatsData(columnStatisticsData);
+    LOG.debug("Ndv estimatation for " + colName + " is "
+        + columnStatisticsData.getDecimalStats().getNumDVs());
+    return statsObj;
+  }
+
+  @Override
+  public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+      int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+      Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
+    int rightBorderInd = numParts;
+    DecimalColumnStatsData extrapolateDecimalData = new DecimalColumnStatsData();
+    Map<String, DecimalColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
+    for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
+      extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDecimalStats());
+    }
+    List<Map.Entry<String, DecimalColumnStatsData>> list = new LinkedList<Map.Entry<String, DecimalColumnStatsData>>(
+        extractedAdjustedStatsMap.entrySet());
+    // get the lowValue
+    Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
+      public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
+          Map.Entry<String, DecimalColumnStatsData> o2) {
+        return o1.getValue().getLowValue().compareTo(o2.getValue().getLowValue());
+      }
+    });
+    double minInd = adjustedIndexMap.get(list.get(0).getKey());
+    double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+    double lowValue = 0;
+    double min = MetaStoreUtils.decimalToDouble(list.get(0).getValue().getLowValue());
+    double max = MetaStoreUtils.decimalToDouble(list.get(list.size() - 1).getValue().getLowValue());
+    if (minInd == maxInd) {
+      lowValue = min;
+    } else if (minInd < maxInd) {
+      // left border is the min
+      lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
+    } else {
+      // right border is the min
+      lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+    }
+
+    // get the highValue
+    Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
+      public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
+          Map.Entry<String, DecimalColumnStatsData> o2) {
+        return o1.getValue().getHighValue().compareTo(o2.getValue().getHighValue());
+      }
+    });
+    minInd = adjustedIndexMap.get(list.get(0).getKey());
+    maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+    double highValue = 0;
+    min = MetaStoreUtils.decimalToDouble(list.get(0).getValue().getHighValue());
+    max = MetaStoreUtils.decimalToDouble(list.get(list.size() - 1).getValue().getHighValue());
+    if (minInd == maxInd) {
+      highValue = min;
+    } else if (minInd < maxInd) {
+      // right border is the max
+      highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+    } else {
+      // left border is the max
+      highValue = (min + (max - min) * minInd / (minInd - maxInd));
+    }
+
+    // get the #nulls
+    long numNulls = 0;
+    for (Map.Entry<String, DecimalColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
+      numNulls += entry.getValue().getNumNulls();
+    }
+    // we scale up sumNulls based on the number of partitions
+    numNulls = numNulls * numParts / numPartsWithStats;
+
+    // get the ndv
+    long ndv = 0;
+    long ndvMin = 0;
+    long ndvMax = 0;
+    Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
+      public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
+          Map.Entry<String, DecimalColumnStatsData> o2) {
+        return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
+      }
+    });
+    long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
+    long higherBound = 0;
+    for (Map.Entry<String, DecimalColumnStatsData> entry : list) {
+      higherBound += entry.getValue().getNumDVs();
+    }
+    if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
+      ndv = (long) ((highValue - lowValue) / densityAvg);
+      if (ndv < lowerBound) {
+        ndv = lowerBound;
+      } else if (ndv > higherBound) {
+        ndv = higherBound;
+      }
+    } else {
+      minInd = adjustedIndexMap.get(list.get(0).getKey());
+      maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+      ndvMin = list.get(0).getValue().getNumDVs();
+      ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
+      if (minInd == maxInd) {
+        ndv = ndvMin;
+      } else if (minInd < maxInd) {
+        // right border is the max
+        ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
+      } else {
+        // left border is the max
+        ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
+      }
+    }
+    extrapolateDecimalData.setLowValue(StatObjectConverter.createThriftDecimal(String
+        .valueOf(lowValue)));
+    extrapolateDecimalData.setHighValue(StatObjectConverter.createThriftDecimal(String
+        .valueOf(highValue)));
+    extrapolateDecimalData.setNumNulls(numNulls);
+    extrapolateDecimalData.setNumDVs(ndv);
+    extrapolateData.setDecimalStats(extrapolateDecimalData);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java
new file mode 100644
index 0000000..e55c412
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java
@@ -0,0 +1,345 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implements
+    IExtrapolatePartStatus {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
+
+  @Override
+  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+      List<ColumnStatistics> css) throws MetaException {
+    ColumnStatisticsObj statsObj = null;
+
+    // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+    // bitvectors
+    boolean doAllPartitionContainStats = partNames.size() == css.size();
+    LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
+    NumDistinctValueEstimator ndvEstimator = null;
+    String colType = null;
+    for (ColumnStatistics cs : css) {
+      if (cs.getStatsObjSize() != 1) {
+        throw new MetaException(
+            "The number of columns should be exactly one in aggrStats, but found "
+                + cs.getStatsObjSize());
+      }
+      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+      if (statsObj == null) {
+        colType = cso.getColType();
+        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+            .getStatsData().getSetField());
+      }
+      if (!cso.getStatsData().getDoubleStats().isSetBitVectors()
+          || cso.getStatsData().getDoubleStats().getBitVectors().length() == 0) {
+        ndvEstimator = null;
+        break;
+      } else {
+        // check if all of the bit vectors can merge
+        NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
+            .getNumDistinctValueEstimator(cso.getStatsData().getDoubleStats().getBitVectors());
+        if (ndvEstimator == null) {
+          ndvEstimator = estimator;
+        } else {
+          if (ndvEstimator.canMerge(estimator)) {
+            continue;
+          } else {
+            ndvEstimator = null;
+            break;
+          }
+        }
+      }
+    }
+    if (ndvEstimator != null) {
+      ndvEstimator = NumDistinctValueEstimatorFactory
+          .getEmptyNumDistinctValueEstimator(ndvEstimator);
+    }
+    LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
+    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+    if (doAllPartitionContainStats || css.size() < 2) {
+      DoubleColumnStatsData aggregateData = null;
+      long lowerBound = 0;
+      long higherBound = 0;
+      double densityAvgSum = 0.0;
+      for (ColumnStatistics cs : css) {
+        ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+        DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
+        lowerBound = Math.max(lowerBound, newData.getNumDVs());
+        higherBound += newData.getNumDVs();
+        densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+        if (ndvEstimator != null) {
+          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+              .getNumDistinctValueEstimator(newData.getBitVectors()));
+        }
+        if (aggregateData == null) {
+          aggregateData = newData.deepCopy();
+        } else {
+          aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+          aggregateData
+              .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+          aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+          aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+        }
+      }
+      if (ndvEstimator != null) {
+        // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
+        // use uniform distribution assumption because we can merge bitvectors
+        // to get a good estimation.
+        aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+      } else {
+        long estimation;
+        if (useDensityFunctionForNDVEstimation) {
+          // We have estimation, lowerbound and higherbound. We use estimation
+          // if it is between lowerbound and higherbound.
+          double densityAvg = densityAvgSum / partNames.size();
+          estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
+          if (estimation < lowerBound) {
+            estimation = lowerBound;
+          } else if (estimation > higherBound) {
+            estimation = higherBound;
+          }
+        } else {
+          estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
+        }
+        aggregateData.setNumDVs(estimation);
+      }
+      columnStatisticsData.setDoubleStats(aggregateData);
+    } else {
+      // we need extrapolation
+      LOG.debug("start extrapolation for " + colName);
+      Map<String, Integer> indexMap = new HashMap<String, Integer>();
+      for (int index = 0; index < partNames.size(); index++) {
+        indexMap.put(partNames.get(index), index);
+      }
+      Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
+      Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
+      // while we scan the css, we also get the densityAvg, lowerbound and
+      // higerbound when useDensityFunctionForNDVEstimation is true.
+      double densityAvgSum = 0.0;
+      if (ndvEstimator == null) {
+        // if not every partition uses bitvector for ndv, we just fall back to
+        // the traditional extrapolation methods.
+        for (ColumnStatistics cs : css) {
+          String partName = cs.getStatsDesc().getPartName();
+          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+          DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
+          if (useDensityFunctionForNDVEstimation) {
+            densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+          }
+          adjustedIndexMap.put(partName, (double) indexMap.get(partName));
+          adjustedStatsMap.put(partName, cso.getStatsData());
+        }
+      } else {
+        // we first merge all the adjacent bitvectors that we could merge and
+        // derive new partition names and index.
+        StringBuilder pseudoPartName = new StringBuilder();
+        double pseudoIndexSum = 0;
+        int length = 0;
+        int curIndex = -1;
+        DoubleColumnStatsData aggregateData = null;
+        for (ColumnStatistics cs : css) {
+          String partName = cs.getStatsDesc().getPartName();
+          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+          DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
+          // newData.isSetBitVectors() should be true for sure because we
+          // already checked it before.
+          if (indexMap.get(partName) != curIndex) {
+            // There is bitvector, but it is not adjacent to the previous ones.
+            if (length > 0) {
+              // we have to set ndv
+              adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+              aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+              ColumnStatisticsData csd = new ColumnStatisticsData();
+              csd.setDoubleStats(aggregateData);
+              adjustedStatsMap.put(pseudoPartName.toString(), csd);
+              if (useDensityFunctionForNDVEstimation) {
+                densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+              }
+              // reset everything
+              pseudoPartName = new StringBuilder();
+              pseudoIndexSum = 0;
+              length = 0;
+              ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
+            }
+            aggregateData = null;
+          }
+          curIndex = indexMap.get(partName);
+          pseudoPartName.append(partName);
+          pseudoIndexSum += curIndex;
+          length++;
+          curIndex++;
+          if (aggregateData == null) {
+            aggregateData = newData.deepCopy();
+          } else {
+            aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+            aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
+                newData.getHighValue()));
+            aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+          }
+          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+              .getNumDistinctValueEstimator(newData.getBitVectors()));
+        }
+        if (length > 0) {
+          // we have to set ndv
+          adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+          aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+          ColumnStatisticsData csd = new ColumnStatisticsData();
+          csd.setDoubleStats(aggregateData);
+          adjustedStatsMap.put(pseudoPartName.toString(), csd);
+          if (useDensityFunctionForNDVEstimation) {
+            densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+          }
+        }
+      }
+      extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
+          adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
+    }
+    LOG.debug("Ndv estimatation for " + colName + " is "
+        + columnStatisticsData.getDoubleStats().getNumDVs());
+    statsObj.setStatsData(columnStatisticsData);
+    return statsObj;
+  }
+
+  @Override
+  public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+      int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+      Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
+    int rightBorderInd = numParts;
+    DoubleColumnStatsData extrapolateDoubleData = new DoubleColumnStatsData();
+    Map<String, DoubleColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
+    for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
+      extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDoubleStats());
+    }
+    List<Map.Entry<String, DoubleColumnStatsData>> list = new LinkedList<Map.Entry<String, DoubleColumnStatsData>>(
+        extractedAdjustedStatsMap.entrySet());
+    // get the lowValue
+    Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
+      public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
+          Map.Entry<String, DoubleColumnStatsData> o2) {
+        return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
+      }
+    });
+    double minInd = adjustedIndexMap.get(list.get(0).getKey());
+    double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+    double lowValue = 0;
+    double min = list.get(0).getValue().getLowValue();
+    double max = list.get(list.size() - 1).getValue().getLowValue();
+    if (minInd == maxInd) {
+      lowValue = min;
+    } else if (minInd < maxInd) {
+      // left border is the min
+      lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
+    } else {
+      // right border is the min
+      lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+    }
+
+    // get the highValue
+    Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
+      public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
+          Map.Entry<String, DoubleColumnStatsData> o2) {
+        return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
+      }
+    });
+    minInd = adjustedIndexMap.get(list.get(0).getKey());
+    maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+    double highValue = 0;
+    min = list.get(0).getValue().getHighValue();
+    max = list.get(list.size() - 1).getValue().getHighValue();
+    if (minInd == maxInd) {
+      highValue = min;
+    } else if (minInd < maxInd) {
+      // right border is the max
+      highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+    } else {
+      // left border is the max
+      highValue = (min + (max - min) * minInd / (minInd - maxInd));
+    }
+
+    // get the #nulls
+    long numNulls = 0;
+    for (Map.Entry<String, DoubleColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
+      numNulls += entry.getValue().getNumNulls();
+    }
+    // we scale up sumNulls based on the number of partitions
+    numNulls = numNulls * numParts / numPartsWithStats;
+
+    // get the ndv
+    long ndv = 0;
+    long ndvMin = 0;
+    long ndvMax = 0;
+    Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
+      public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
+          Map.Entry<String, DoubleColumnStatsData> o2) {
+        return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
+      }
+    });
+    long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
+    long higherBound = 0;
+    for (Map.Entry<String, DoubleColumnStatsData> entry : list) {
+      higherBound += entry.getValue().getNumDVs();
+    }
+    if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
+      ndv = (long) ((highValue - lowValue) / densityAvg);
+      if (ndv < lowerBound) {
+        ndv = lowerBound;
+      } else if (ndv > higherBound) {
+        ndv = higherBound;
+      }
+    } else {
+      minInd = adjustedIndexMap.get(list.get(0).getKey());
+      maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+      ndvMin = list.get(0).getValue().getNumDVs();
+      ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
+      if (minInd == maxInd) {
+        ndv = ndvMin;
+      } else if (minInd < maxInd) {
+        // right border is the max
+        ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
+      } else {
+        // left border is the max
+        ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
+      }
+    }
+    extrapolateDoubleData.setLowValue(lowValue);
+    extrapolateDoubleData.setHighValue(highValue);
+    extrapolateDoubleData.setNumNulls(numNulls);
+    extrapolateDoubleData.setNumDVs(ndv);
+    extrapolateData.setDoubleStats(extrapolateDoubleData);
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/IExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/IExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/IExtrapolatePartStatus.java
new file mode 100644
index 0000000..acf679e
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/IExtrapolatePartStatus.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+
+public interface IExtrapolatePartStatus {
+  // The following function will extrapolate the stats when the column stats of
+  // some partitions are missing.
+  /**
+   * @param extrapolateData
+   *          it will carry back the specific stats, e.g., DOUBLE_STATS or
+   *          LONG_STATS
+   * @param numParts
+   *          the total number of partitions
+   * @param numPartsWithStats
+   *          the number of partitions that have stats
+   * @param adjustedIndexMap
+   *          the partition name to index map
+   * @param adjustedStatsMap
+   *          the partition name to its stats map
+   * @param densityAvg
+   *          the average of ndv density, which is useful when
+   *          useDensityFunctionForNDVEstimation is true.
+   */
+  public abstract void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+      int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+      Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg);
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java
new file mode 100644
index 0000000..2ee09f3
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
+    IExtrapolatePartStatus {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
+
+  @Override
+  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+      List<ColumnStatistics> css) throws MetaException {
+    ColumnStatisticsObj statsObj = null;
+
+    // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+    // bitvectors
+    boolean doAllPartitionContainStats = partNames.size() == css.size();
+    LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
+    NumDistinctValueEstimator ndvEstimator = null;
+    String colType = null;
+    for (ColumnStatistics cs : css) {
+      if (cs.getStatsObjSize() != 1) {
+        throw new MetaException(
+            "The number of columns should be exactly one in aggrStats, but found "
+                + cs.getStatsObjSize());
+      }
+      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+      if (statsObj == null) {
+        colType = cso.getColType();
+        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+            .getStatsData().getSetField());
+      }
+      if (!cso.getStatsData().getLongStats().isSetBitVectors()
+          || cso.getStatsData().getLongStats().getBitVectors().length() == 0) {
+        ndvEstimator = null;
+        break;
+      } else {
+        // check if all of the bit vectors can merge
+        NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
+            .getNumDistinctValueEstimator(cso.getStatsData().getLongStats().getBitVectors());
+        if (ndvEstimator == null) {
+          ndvEstimator = estimator;
+        } else {
+          if (ndvEstimator.canMerge(estimator)) {
+            continue;
+          } else {
+            ndvEstimator = null;
+            break;
+          }
+        }
+      }
+    }
+    if (ndvEstimator != null) {
+      ndvEstimator = NumDistinctValueEstimatorFactory
+          .getEmptyNumDistinctValueEstimator(ndvEstimator);
+    }
+    LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
+    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+    if (doAllPartitionContainStats || css.size() < 2) {
+      LongColumnStatsData aggregateData = null;
+      long lowerBound = 0;
+      long higherBound = 0;
+      double densityAvgSum = 0.0;
+      for (ColumnStatistics cs : css) {
+        ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+        LongColumnStatsData newData = cso.getStatsData().getLongStats();
+        lowerBound = Math.max(lowerBound, newData.getNumDVs());
+        higherBound += newData.getNumDVs();
+        densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+        if (ndvEstimator != null) {
+          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+              .getNumDistinctValueEstimator(newData.getBitVectors()));
+        }
+        if (aggregateData == null) {
+          aggregateData = newData.deepCopy();
+        } else {
+          aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+          aggregateData
+              .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+          aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+          aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+        }
+      }
+      if (ndvEstimator != null) {
+        // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
+        // use uniform distribution assumption because we can merge bitvectors
+        // to get a good estimation.
+        aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+      } else {
+        long estimation;
+        if (useDensityFunctionForNDVEstimation) {
+          // We have estimation, lowerbound and higherbound. We use estimation
+          // if it is between lowerbound and higherbound.
+          double densityAvg = densityAvgSum / partNames.size();
+          estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
+          if (estimation < lowerBound) {
+            estimation = lowerBound;
+          } else if (estimation > higherBound) {
+            estimation = higherBound;
+          }
+        } else {
+          estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
+        }
+        aggregateData.setNumDVs(estimation);
+      }
+      columnStatisticsData.setLongStats(aggregateData);
+    } else {
+      // we need extrapolation
+      LOG.debug("start extrapolation for " + colName);
+
+      Map<String, Integer> indexMap = new HashMap<String, Integer>();
+      for (int index = 0; index < partNames.size(); index++) {
+        indexMap.put(partNames.get(index), index);
+      }
+      Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
+      Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
+      // while we scan the css, we also get the densityAvg, lowerbound and
+      // higerbound when useDensityFunctionForNDVEstimation is true.
+      double densityAvgSum = 0.0;
+      if (ndvEstimator == null) {
+        // if not every partition uses bitvector for ndv, we just fall back to
+        // the traditional extrapolation methods.
+        for (ColumnStatistics cs : css) {
+          String partName = cs.getStatsDesc().getPartName();
+          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+          LongColumnStatsData newData = cso.getStatsData().getLongStats();
+          if (useDensityFunctionForNDVEstimation) {
+            densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+          }
+          adjustedIndexMap.put(partName, (double) indexMap.get(partName));
+          adjustedStatsMap.put(partName, cso.getStatsData());
+        }
+      } else {
+        // we first merge all the adjacent bitvectors that we could merge and
+        // derive new partition names and index.
+        StringBuilder pseudoPartName = new StringBuilder();
+        double pseudoIndexSum = 0;
+        int length = 0;
+        int curIndex = -1;
+        LongColumnStatsData aggregateData = null;
+        for (ColumnStatistics cs : css) {
+          String partName = cs.getStatsDesc().getPartName();
+          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+          LongColumnStatsData newData = cso.getStatsData().getLongStats();
+          // newData.isSetBitVectors() should be true for sure because we
+          // already checked it before.
+          if (indexMap.get(partName) != curIndex) {
+            // There is bitvector, but it is not adjacent to the previous ones.
+            if (length > 0) {
+              // we have to set ndv
+              adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+              aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+              ColumnStatisticsData csd = new ColumnStatisticsData();
+              csd.setLongStats(aggregateData);
+              adjustedStatsMap.put(pseudoPartName.toString(), csd);
+              if (useDensityFunctionForNDVEstimation) {
+                densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+              }
+              // reset everything
+              pseudoPartName = new StringBuilder();
+              pseudoIndexSum = 0;
+              length = 0;
+              ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
+            }
+            aggregateData = null;
+          }
+          curIndex = indexMap.get(partName);
+          pseudoPartName.append(partName);
+          pseudoIndexSum += curIndex;
+          length++;
+          curIndex++;
+          if (aggregateData == null) {
+            aggregateData = newData.deepCopy();
+          } else {
+            aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+            aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
+                newData.getHighValue()));
+            aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+          }
+          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+              .getNumDistinctValueEstimator(newData.getBitVectors()));
+        }
+        if (length > 0) {
+          // we have to set ndv
+          adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+          aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+          ColumnStatisticsData csd = new ColumnStatisticsData();
+          csd.setLongStats(aggregateData);
+          adjustedStatsMap.put(pseudoPartName.toString(), csd);
+          if (useDensityFunctionForNDVEstimation) {
+            densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+          }
+        }
+      }
+      extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
+          adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
+    }
+    statsObj.setStatsData(columnStatisticsData);
+    LOG.debug("Ndv estimatation for " + colName + " is "
+        + columnStatisticsData.getLongStats().getNumDVs());
+    return statsObj;
+  }
+
+  @Override
+  public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+      int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+      Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
+    int rightBorderInd = numParts;
+    LongColumnStatsData extrapolateLongData = new LongColumnStatsData();
+    Map<String, LongColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
+    for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
+      extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getLongStats());
+    }
+    List<Map.Entry<String, LongColumnStatsData>> list = new LinkedList<Map.Entry<String, LongColumnStatsData>>(
+        extractedAdjustedStatsMap.entrySet());
+    // get the lowValue
+    Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
+      public int compare(Map.Entry<String, LongColumnStatsData> o1,
+          Map.Entry<String, LongColumnStatsData> o2) {
+        return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
+      }
+    });
+    double minInd = adjustedIndexMap.get(list.get(0).getKey());
+    double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+    long lowValue = 0;
+    long min = list.get(0).getValue().getLowValue();
+    long max = list.get(list.size() - 1).getValue().getLowValue();
+    if (minInd == maxInd) {
+      lowValue = min;
+    } else if (minInd < maxInd) {
+      // left border is the min
+      lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
+    } else {
+      // right border is the min
+      lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+    }
+
+    // get the highValue
+    Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
+      public int compare(Map.Entry<String, LongColumnStatsData> o1,
+          Map.Entry<String, LongColumnStatsData> o2) {
+        return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
+      }
+    });
+    minInd = adjustedIndexMap.get(list.get(0).getKey());
+    maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+    long highValue = 0;
+    min = list.get(0).getValue().getHighValue();
+    max = list.get(list.size() - 1).getValue().getHighValue();
+    if (minInd == maxInd) {
+      highValue = min;
+    } else if (minInd < maxInd) {
+      // right border is the max
+      highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+    } else {
+      // left border is the max
+      highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
+    }
+
+    // get the #nulls
+    long numNulls = 0;
+    for (Map.Entry<String, LongColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
+      numNulls += entry.getValue().getNumNulls();
+    }
+    // we scale up sumNulls based on the number of partitions
+    numNulls = numNulls * numParts / numPartsWithStats;
+
+    // get the ndv
+    long ndv = 0;
+    Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
+      public int compare(Map.Entry<String, LongColumnStatsData> o1,
+          Map.Entry<String, LongColumnStatsData> o2) {
+        return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
+      }
+    });
+    long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
+    long higherBound = 0;
+    for (Map.Entry<String, LongColumnStatsData> entry : list) {
+      higherBound += entry.getValue().getNumDVs();
+    }
+    if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
+      ndv = (long) ((highValue - lowValue) / densityAvg);
+      if (ndv < lowerBound) {
+        ndv = lowerBound;
+      } else if (ndv > higherBound) {
+        ndv = higherBound;
+      }
+    } else {
+      minInd = adjustedIndexMap.get(list.get(0).getKey());
+      maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+      min = list.get(0).getValue().getNumDVs();
+      max = list.get(list.size() - 1).getValue().getNumDVs();
+      if (minInd == maxInd) {
+        ndv = min;
+      } else if (minInd < maxInd) {
+        // right border is the max
+        ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+      } else {
+        // left border is the max
+        ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
+      }
+    }
+    extrapolateLongData.setLowValue(lowValue);
+    extrapolateLongData.setHighValue(highValue);
+    extrapolateLongData.setNumNulls(numNulls);
+    extrapolateLongData.setNumDVs(ndv);
+    extrapolateData.setLongStats(extrapolateLongData);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
new file mode 100644
index 0000000..4c2d1bc
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+
+public class BinaryColumnStatsMerger extends ColumnStatsMerger {
+
+  @Override
+  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+    BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats();
+    BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats();
+    aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
+    aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
+    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
new file mode 100644
index 0000000..8e50153
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+
+public class BooleanColumnStatsMerger extends ColumnStatsMerger {
+
+  @Override
+  public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+    BooleanColumnStatsData aggregateData = aggregateColStats.getStatsData().getBooleanStats();
+    BooleanColumnStatsData newData = newColStats.getStatsData().getBooleanStats();
+    aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
+    aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
+    aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
new file mode 100644
index 0000000..474d4dd
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class ColumnStatsMerger {
+  protected final Logger LOG = LoggerFactory.getLogger(ColumnStatsMerger.class.getName());
+
+  public abstract void merge(ColumnStatisticsObj aggregateColStats,
+      ColumnStatisticsObj newColStats);
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java
new file mode 100644
index 0000000..0ce1847
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+
+public class ColumnStatsMergerFactory {
+
+  private ColumnStatsMergerFactory() {
+  }
+
+  public static ColumnStatsMerger getColumnStatsMerger(ColumnStatisticsObj statsObjNew,
+      ColumnStatisticsObj statsObjOld) {
+    ColumnStatsMerger agg;
+    _Fields typeNew = statsObjNew.getStatsData().getSetField();
+    _Fields typeOld = statsObjOld.getStatsData().getSetField();
+    // make sure that they have the same type
+    typeNew = typeNew == typeOld ? typeNew : null;
+    switch (typeNew) {
+    case BOOLEAN_STATS:
+      agg = new BooleanColumnStatsMerger();
+      break;
+    case LONG_STATS: {
+      agg = new LongColumnStatsMerger();
+      break;
+    }
+    case DOUBLE_STATS: {
+      agg = new DoubleColumnStatsMerger();
+      break;
+    }
+    case STRING_STATS: {
+      agg = new StringColumnStatsMerger();
+      break;
+    }
+    case BINARY_STATS:
+      agg = new BinaryColumnStatsMerger();
+      break;
+    case DECIMAL_STATS: {
+      agg = new DecimalColumnStatsMerger();
+      break;
+    }
+    case DATE_STATS: {
+      agg = new DateColumnStatsMerger();
+      break;
+    }
+    default:
+      throw new IllegalArgumentException("Unknown stats type " + typeNew.toString());
+    }
+    return agg;
+  }
+
+  public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
+    ColumnStatisticsObj cso = new ColumnStatisticsObj();
+    ColumnStatisticsData csd = new ColumnStatisticsData();
+    cso.setColName(colName);
+    cso.setColType(colType);
+    switch (type) {
+    case BOOLEAN_STATS:
+      csd.setBooleanStats(new BooleanColumnStatsData());
+      break;
+
+    case LONG_STATS:
+      csd.setLongStats(new LongColumnStatsData());
+      break;
+
+    case DOUBLE_STATS:
+      csd.setDoubleStats(new DoubleColumnStatsData());
+      break;
+
+    case STRING_STATS:
+      csd.setStringStats(new StringColumnStatsData());
+      break;
+
+    case BINARY_STATS:
+      csd.setBinaryStats(new BinaryColumnStatsData());
+      break;
+
+    case DECIMAL_STATS:
+      csd.setDecimalStats(new DecimalColumnStatsData());
+      break;
+
+    case DATE_STATS:
+      csd.setDateStats(new DateColumnStatsData());
+      break;
+
+    default:
+      throw new IllegalArgumentException("Unknown stats type");
+    }
+
+    cso.setStatsData(csd);
+    return cso;
+  }
+
+}


[16/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
deleted file mode 100644
index aefafe0..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
-import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
-import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
-import org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics;
-import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-
-/**
- * Test HMS Metrics on HBase Metastore
- */
-public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {
-
-  @BeforeClass
-  public static void startup() throws Exception {
-    HBaseIntegrationTests.startMiniCluster();
-  }
-
-  @AfterClass
-  public static void shutdown() throws Exception {
-    HBaseIntegrationTests.shutdownMiniCluster();
-  }
-
-  @Before
-  public void before() throws IOException {
-    HBaseReadWrite.setConf(conf);
-    conf = new HiveConf();
-    conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
-      "org.apache.hadoop.hive.metastore.hbase.HBaseStore");
-    conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true);
-    conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
-
-    conf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true);
-    conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
-    SessionState.start(new CliSessionState(conf));
-    driver = new Driver(conf);
-  }
-
-  @Test
-  public void testMetaDataCounts() throws Exception {
-    //1 databases created
-    driver.run("create database testdb1");
-
-    //4 tables
-    driver.run("create table testtbl1 (key string)");
-    driver.run("create table testtblpart (key string) partitioned by (partkey string)");
-    driver.run("use testdb1");
-    driver.run("create table testtbl2 (key string)");
-    driver.run("create table testtblpart2 (key string) partitioned by (partkey string)");
-
-    //6 partitions
-    driver.run("alter table default.testtblpart add partition (partkey='a')");
-    driver.run("alter table default.testtblpart add partition (partkey='b')");
-    driver.run("alter table default.testtblpart add partition (partkey='c')");
-    driver.run("alter table testdb1.testtblpart2 add partition (partkey='a')");
-    driver.run("alter table testdb1.testtblpart2 add partition (partkey='b')");
-    driver.run("alter table testdb1.testtblpart2 add partition (partkey='c')");
-
-
-    //create and drop some additional metadata, to test drop counts.
-    driver.run("create database tempdb");
-    driver.run("use tempdb");
-
-    driver.run("create table delete_by_table (key string) partitioned by (partkey string)");
-    driver.run("alter table delete_by_table add partition (partkey='temp')");
-    driver.run("drop table delete_by_table");
-
-    driver.run("create table delete_by_part (key string) partitioned by (partkey string)");
-    driver.run("alter table delete_by_part add partition (partkey='temp')");
-    driver.run("alter table delete_by_part drop partition (partkey='temp')");
-
-    driver.run("create table delete_by_db (key string) partitioned by (partkey string)");
-    driver.run("alter table delete_by_db add partition (partkey='temp')");
-    driver.run("use default");
-    driver.run("drop database tempdb cascade");
-
-    CodahaleMetrics metrics = (CodahaleMetrics) MetricsFactory.getInstance();
-    String json = metrics.dumpJson();
-    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_DATABASES, 2);
-    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_TABLES, 7);
-    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_PARTITIONS, 9);
-
-    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.DELETE_TOTAL_DATABASES, 1);
-    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.DELETE_TOTAL_TABLES, 3);
-    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.DELETE_TOTAL_PARTITIONS, 3);
-
-
-    //to test initial metadata count metrics.
-    conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, ObjectStore.class.getName());
-    HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("test", conf, false);
-    baseHandler.init();
-    baseHandler.updateMetrics();
-
-    //1 new db + default
-    json = metrics.dumpJson();
-    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_DATABASES, 2);
-    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_TABLES, 4);
-    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_PARTITIONS, 6);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
deleted file mode 100644
index d4966b9..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-
-/**
- * Integration tests with HBase Mini-cluster using actual SQL
- */
-public class TestHBaseMetastoreSql extends HBaseIntegrationTests {
-
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreIntegration.class.getName());
-
-  @BeforeClass
-  public static void startup() throws Exception {
-    HBaseIntegrationTests.startMiniCluster();
-
-  }
-
-  @AfterClass
-  public static void shutdown() throws Exception {
-    HBaseIntegrationTests.shutdownMiniCluster();
-  }
-
-  @Before
-  public void before() throws IOException {
-    setupConnection();
-    setupDriver();
-  }
-
-  @Test
-  public void insertIntoTable() throws Exception {
-    driver.run("create table iit (c int)");
-    CommandProcessorResponse rsp = driver.run("insert into table iit values (3)");
-    Assert.assertEquals(0, rsp.getResponseCode());
-  }
-
-  @Test
-  public void insertIntoPartitionTable() throws Exception {
-    driver.run("create table iipt (c int) partitioned by (ds string)");
-    CommandProcessorResponse rsp =
-        driver.run("insert into table iipt partition(ds) values (1, 'today'), (2, 'yesterday')," +
-            "(3, 'tomorrow')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-  }
-
-  @Test
-  public void database() throws Exception {
-    CommandProcessorResponse rsp = driver.run("create database db");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("set role admin");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    // security doesn't let me change the properties
-    rsp = driver.run("alter database db set dbproperties ('key' = 'value')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("drop database db");
-    Assert.assertEquals(0, rsp.getResponseCode());
-  }
-
-  @Test
-  public void table() throws Exception {
-    driver.run("create table tbl (c int)");
-    CommandProcessorResponse rsp = driver.run("insert into table tbl values (3)");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("select * from tbl");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("alter table tbl set tblproperties ('example', 'true')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("drop table tbl");
-    Assert.assertEquals(0, rsp.getResponseCode());
-  }
-
-  @Test
-  public void partitionedTable() throws Exception {
-    driver.run("create table parttbl (c int) partitioned by (ds string)");
-    CommandProcessorResponse rsp =
-        driver.run("insert into table parttbl partition(ds) values (1, 'today'), (2, 'yesterday')" +
-            ", (3, 'tomorrow')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    // Do it again, to check insert into existing partitions
-    rsp = driver.run("insert into table parttbl partition(ds) values (4, 'today'), (5, 'yesterday')"
-        + ", (6, 'tomorrow')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("insert into table parttbl partition(ds = 'someday') values (1)");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("insert into table parttbl partition(ds = 'someday') values (2)");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("alter table parttbl add partition (ds = 'whenever')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("insert into table parttbl partition(ds = 'whenever') values (2)");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("alter table parttbl touch partition (ds = 'whenever')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    // TODO - Can't do this until getPartitionsByExpr implemented
-    /*
-    rsp = driver.run("alter table parttbl drop partition (ds = 'whenever')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    */
-    rsp = driver.run("select * from parttbl");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("select * from parttbl where ds = 'today'");
-    Assert.assertEquals(0, rsp.getResponseCode());
-  }
-
-  @Test
-  public void role() throws Exception {
-    CommandProcessorResponse rsp = driver.run("set role admin");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("create role role1");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("grant role1 to user fred with admin option");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("create role role2");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("grant role1 to role role2");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("show principals role1");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("show role grant role role1");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("show role grant user " + System.getProperty("user.name"));
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("show roles");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("revoke admin option for role1 from user fred");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("revoke role1 from user fred");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("revoke role1 from role role2");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("show current roles");
-    Assert.assertEquals(0, rsp.getResponseCode());
-
-    rsp = driver.run("drop role role2");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("drop role role1");
-    Assert.assertEquals(0, rsp.getResponseCode());
-  }
-
-  @Test
-  public void grant() throws Exception {
-    CommandProcessorResponse rsp = driver.run("set role admin");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("create role role3");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    driver.run("create table granttbl (c int)");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    driver.run("grant select on granttbl to " + System.getProperty("user.name"));
-    Assert.assertEquals(0, rsp.getResponseCode());
-    driver.run("grant select on granttbl to role3 with grant option");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    driver.run("revoke select on granttbl from " + System.getProperty("user.name"));
-    Assert.assertEquals(0, rsp.getResponseCode());
-    driver.run("revoke grant option for select on granttbl from role3");
-    Assert.assertEquals(0, rsp.getResponseCode());
-  }
-
-  @Test
-  public void describeNonpartitionedTable() throws Exception {
-    CommandProcessorResponse rsp = driver.run("create table alter1(a int, b int)");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("describe extended alter1");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("alter table alter1 set serdeproperties('s1'='9')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("describe extended alter1");
-    Assert.assertEquals(0, rsp.getResponseCode());
-  }
-
-  @Test
-  public void alterRenamePartitioned() throws Exception {
-    driver.run("create table alterrename (c int) partitioned by (ds string)");
-    driver.run("alter table alterrename add partition (ds = 'a')");
-    CommandProcessorResponse rsp = driver.run("describe extended alterrename partition (ds='a')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("alter table alterrename rename to alter_renamed");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("describe extended alter_renamed partition (ds='a')");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("describe extended alterrename partition (ds='a')");
-    Assert.assertEquals(10001, rsp.getResponseCode());
-  }
-
-  @Test
-  public void alterRename() throws Exception {
-    driver.run("create table alterrename1 (c int)");
-    CommandProcessorResponse rsp = driver.run("describe alterrename1");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("alter table alterrename1 rename to alter_renamed1");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("describe alter_renamed1");
-    Assert.assertEquals(0, rsp.getResponseCode());
-    rsp = driver.run("describe alterrename1");
-    Assert.assertEquals(10001, rsp.getResponseCode());
-  }
-
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java
deleted file mode 100644
index c98911a..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java
+++ /dev/null
@@ -1,582 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.HiveObjectType;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-public class TestHBaseSchemaTool extends HBaseIntegrationTests {
-
-  private final String lsep = System.getProperty("line.separator");
-
-  @BeforeClass
-  public static void startup() throws Exception {
-    HBaseIntegrationTests.startMiniCluster();
-  }
-
-  @AfterClass
-  public static void shutdown() throws Exception {
-    HBaseIntegrationTests.shutdownMiniCluster();
-  }
-
-  @Before
-  public void setup() throws IOException {
-    setupHBaseStore();
-  }
-
-  @Test
-  public void listTables() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(true, null, null, null, conf, out, err);
-    Assert.assertEquals(StringUtils.join(HBaseReadWrite.tableNames, lsep) + lsep,
-        outStr.toString());
-  }
-
-  @Test
-  public void bogusTable() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, "nosuch", null, null, conf, out, err);
-    Assert.assertEquals("Unknown table: nosuch" + lsep, errStr.toString());
-  }
-
-  @Test
-  public void noSuchDb() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.DB_TABLE, "nosuch", null, conf, out, err);
-    Assert.assertEquals("No such database: nosuch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noMatchingDb() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.DB_TABLE, null, "nomatch", conf, out, err);
-    Assert.assertEquals("No matching database: nomatch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noSuchRole() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.ROLE_TABLE, "nosuch", null, conf, out, err);
-    Assert.assertEquals("No such role: nosuch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noMatchingRole() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.ROLE_TABLE, null, "nomatch", conf, out, err);
-    Assert.assertEquals("No matching role: nomatch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noSuchUser() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, "nosuch", null, conf, out, err);
-    Assert.assertEquals("No such user: nosuch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noMatchingUser() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, null, "nomatch", conf, out, err);
-    Assert.assertEquals("No matching user: nomatch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noSuchFunction() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.FUNC_TABLE, "nosuch", null, conf, out,  err);
-    Assert.assertEquals("No such function: nosuch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noMatchingFunction() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.FUNC_TABLE, null, "nomatch", conf, out,
-        err);
-    Assert.assertEquals("No matching function: nomatch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noSuchTable() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.TABLE_TABLE, "nosuch", null, conf, out, err);
-    Assert.assertEquals("No such table: nosuch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noMatchingTable() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.TABLE_TABLE, null, "nomatch", conf, out, err);
-    Assert.assertEquals("No matching table: nomatch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noSuchPart() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, "nosuch", null, conf, out, err);
-    Assert.assertEquals("No such partition: nosuch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noSuchPartValidFormat() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-    // Test with something that looks like a valid entry
-    new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, "default.nosuch.nosuch", null, conf,
-        out, err);
-    Assert.assertEquals("No such partition: default.nosuch.nosuch" + lsep, outStr.toString());
-  }
-
-
-  @Test
-  public void noMatchingPart() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, null, "nomatch", conf, out, err);
-    Assert.assertEquals("No matching partition: nomatch" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noMatchingPartValidFormat() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, null, "nomatch.a.b", conf, out, err);
-    Assert.assertEquals("No matching partition: nomatch.a.b" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void noSuchStorageDescriptor() throws Exception {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    // Strangly enough things don't come back quite the same when going through the Base64
-    // encode/decode.
-    new HBaseSchemaTool().go(false, HBaseReadWrite.SD_TABLE, "nosuch", null, conf, out, err);
-    Assert.assertEquals("No such storage descriptor: nosucg" + lsep, outStr.toString());
-  }
-
-  @Test
-  public void oneMondoTest() throws Exception {
-    // This is a pain to do in one big test, but we have to control the order so that we have tests
-    // without dbs, etc.
-    HBaseSchemaTool tool = new HBaseSchemaTool();
-
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    // This needs to be up front before we create any tables or partitions
-    tool.go(false, HBaseReadWrite.SD_TABLE, null, "whatever", conf, out, err);
-    Assert.assertEquals("No storage descriptors" + lsep, outStr.toString());
-
-    // This one needs to be up front too
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.SEQUENCES_TABLE, null, "whatever", conf, out, err);
-    Assert.assertEquals("No sequences" + lsep, outStr.toString());
-
-    // Create some databases
-    String[] dbNames = new String[3];
-    for (int i = 0; i < dbNames.length; i++) {
-      dbNames[i] = "db" + i;
-      Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters);
-      store.createDatabase(db);
-    }
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.DB_TABLE, "db0", null, conf, out, err);
-    Assert.assertEquals("{\"name\":\"db0\",\"description\":\"no description\"," +
-        "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep, outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.DB_TABLE, null, ".*", conf, out, err);
-    Assert.assertEquals("{\"name\":\"db0\",\"description\":\"no description\"," +
-          "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep +
-          "{\"name\":\"db1\",\"description\":\"no description\"," +
-          "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep +
-          "{\"name\":\"db2\",\"description\":\"no description\"," +
-          "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep,
-        outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.DB_TABLE, null, "db[12]", conf, out, err);
-    Assert.assertEquals("{\"name\":\"db1\",\"description\":\"no description\"," +
-            "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep +
-            "{\"name\":\"db2\",\"description\":\"no description\"," +
-            "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep,
-        outStr.toString());
-
-    String[] roleNames = new String[2];
-    for (int i = 0; i < roleNames.length; i++) {
-      roleNames[i] = "role" + i;
-      store.addRole(roleNames[i], "me");
-    }
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.ROLE_TABLE, null, "role.", conf, out, err);
-    Assert.assertEquals("{\"roleName\":\"role0\",\"createTime\":now,\"ownerName\":\"me\"}" +
-        lsep +  "{\"roleName\":\"role1\",\"createTime\":now,\"ownerName\":\"me\"}" + lsep,
-        outStr.toString().replaceAll("createTime\":[0-9]+", "createTime\":now"));
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.ROLE_TABLE, "role1", null, conf, out, err);
-    Assert.assertEquals("{\"roleName\":\"role1\",\"createTime\":now,\"ownerName\":\"me\"}" + lsep,
-        outStr.toString().replaceAll("createTime\":[0-9]+", "createTime\":now"));
-
-    Role role1 = store.getRole("role1");
-    store.grantRole(role1, "fred", PrincipalType.USER, "me", PrincipalType.USER, false);
-    store.grantRole(role1, "joanne", PrincipalType.USER, "me", PrincipalType.USER, false);
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, null, ".*", conf, out, err);
-    Assert.assertEquals("fred: role1" + lsep + "joanne: role1" + lsep, outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, "joanne", null, conf, out, err);
-    Assert.assertEquals("role1" + lsep, outStr.toString());
-
-    String[] funcNames = new String[3];
-    for (int i = 0; i < funcNames.length; i++) {
-      funcNames[i] = "func" + i;
-      Function function = new Function(funcNames[i], "db1", "Function", "me", PrincipalType.USER, 0,
-          FunctionType.JAVA, null);
-      store.createFunction(function);
-    }
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.FUNC_TABLE, "db1.func0", null, conf, out, err);
-    Assert.assertEquals("{\"functionName\":\"func0\",\"dbName\":\"db1\"," +
-        "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
-        "\"functionType\":1}" + lsep, outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.FUNC_TABLE, null, ".*", conf, out, err);
-    Assert.assertEquals("{\"functionName\":\"func0\",\"dbName\":\"db1\"," +
-        "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
-        "\"functionType\":1}" + lsep +
-        "{\"functionName\":\"func1\",\"dbName\":\"db1\"," +
-        "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
-        "\"functionType\":1}" + lsep +
-        "{\"functionName\":\"func2\",\"dbName\":\"db1\"," +
-        "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
-        "\"functionType\":1}" + lsep, outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.FUNC_TABLE, null, "db1.func[12]", conf, out, err);
-    Assert.assertEquals("{\"functionName\":\"func1\",\"dbName\":\"db1\"," +
-        "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
-        "\"functionType\":1}" + lsep +
-        "{\"functionName\":\"func2\",\"dbName\":\"db1\"," +
-        "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
-        "\"functionType\":1}" + lsep, outStr.toString());
-
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.GLOBAL_PRIVS_TABLE, null, null, conf, out, err);
-    Assert.assertEquals("No global privileges" + lsep, outStr.toString());
-
-    List<HiveObjectPrivilege> privileges = new ArrayList<>();
-    HiveObjectRef hiveObjRef = new HiveObjectRef(HiveObjectType.GLOBAL, "db0", "tab0", null,
-        null);
-    PrivilegeGrantInfo grantInfo =
-        new PrivilegeGrantInfo("read", 0, "me", PrincipalType.USER, false);
-    HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, "user", PrincipalType.USER,
-        grantInfo);
-    privileges.add(hop);
-
-    grantInfo = new PrivilegeGrantInfo("create", 0, "me", PrincipalType.USER, true);
-    hop = new HiveObjectPrivilege(hiveObjRef, "user", PrincipalType.USER, grantInfo);
-    privileges.add(hop);
-
-    PrivilegeBag pBag = new PrivilegeBag(privileges);
-    store.grantPrivileges(pBag);
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.GLOBAL_PRIVS_TABLE, null, null, conf, out, err);
-    Assert.assertEquals(
-        "{\"userPrivileges\":{\"user\":[{\"privilege\":\"read\",\"createTime\":0," +
-            "\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":0},{\"privilege\":\"create\"," +
-            "\"createTime\":0,\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":1}]}}" + lsep,
-        outStr.toString());
-
-
-    String[] tableNames = new String[3];
-    for (int i = 0; i < tableNames.length; i++) {
-      tableNames[i] = "tab" + i;
-      StorageDescriptor sd = new StorageDescriptor(Arrays.asList(new FieldSchema("col1", "int",
-          ""), new FieldSchema("col2", "varchar(32)", "")),
-          "/tmp", null, null, false, 0, null,  null, null, Collections.<String, String>emptyMap());
-      Table tab = new Table(tableNames[i], dbNames[0], "me", 0, 0, 0, sd,
-          Arrays.asList(new FieldSchema("pcol1", "string", ""),
-              new FieldSchema("pcol2", "string", "")),
-          Collections.<String, String>emptyMap(), null, null, null);
-      store.createTable(tab);
-    }
-
-    ColumnStatisticsDesc tableStatsDesc = new ColumnStatisticsDesc(false, "db0", "tab0");
-    ColumnStatisticsData tcsd = new ColumnStatisticsData();
-    LongColumnStatsData tlcsd = new LongColumnStatsData(1, 2);
-    tlcsd.setLowValue(-95);
-    tlcsd.setHighValue(95);
-    tcsd.setLongStats(tlcsd);
-    ColumnStatisticsData tcsd2 = new ColumnStatisticsData();
-    tcsd2.setStringStats(new StringColumnStatsData(97, 18.78, 29, 397));
-    List<ColumnStatisticsObj> tcsos = Arrays.asList(
-        new ColumnStatisticsObj("col1", "int", tcsd),
-        new ColumnStatisticsObj("col2", "varchar(32)", tcsd2));
-    ColumnStatistics tStatObj = new ColumnStatistics(tableStatsDesc, tcsos);
-    store.updateTableColumnStatistics(tStatObj);
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.TABLE_TABLE, "db0.tab1", null, conf, out, err);
-    Assert.assertEquals("{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\"," +
-        "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," +
-        "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"}," +
-        "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{}," +
-        "\"tableType\":\"\",\"rewriteEnabled\":0} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep,
-        outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.TABLE_TABLE, null, "db0.*", conf, out, err);
-    Assert.assertEquals("{\"tableName\":\"tab0\",\"dbName\":\"db0\",\"owner\":\"me\"," +
-        "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," +
-        "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"}," +
-        "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{\"COLUMN_STATS_ACCURATE\":\"{\\\"COLUMN_STATS\\\":{\\\"col1\\\":\\\"true\\\",\\\"col2\\\":\\\"true\\\"}}\"}," +
-            "\"tableType\":\"\",\"rewriteEnabled\":0} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column " +
-            "col1: {\"colName\":\"col1\",\"colType\":\"int\"," +
-            "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," +
-            "\"numDVs\":2,\"bitVectors\":\"\"}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," +
-            "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78," +
-        "\"numNulls\":29,\"numDVs\":397,\"bitVectors\":\"\"}}}" + lsep +
-        "{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0," +
-        "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\"," +
-            "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\"," +
-            "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\",\"rewriteEnabled\":0} sdHash: " +
-            "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep +
-        "{\"tableName\":\"tab2\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0," +
-        "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\"," +
-        "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\"," +
-        "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\",\"rewriteEnabled\":0} sdHash: " +
-        "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep, outStr.toString());
-
-    List<List<String>> partVals = Arrays.asList(Arrays.asList("a", "b"), Arrays.asList("c", "d"));
-    for (List<String> pv : partVals) {
-      StorageDescriptor sd = new StorageDescriptor(Arrays.asList(new FieldSchema("col1", "int",
-          ""), new FieldSchema("col2", "varchar(32)", "")),
-          "/tmp", null, null, false, 0, null,  null, null, Collections.<String, String>emptyMap());
-      Partition p = new Partition(pv, "db0", "tab1", 0, 0, sd, Collections.<String, String>emptyMap());
-      store.addPartition(p);
-    }
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.PART_TABLE, "db0.tab1.a.b", null, conf, out, err);
-    Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
-        "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: " +
-        "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep,  outStr.toString());
-
-    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "db0", "tab1");
-    statsDesc.setPartName("pcol1=c/pcol2=d");
-    ColumnStatisticsData csd1 = new ColumnStatisticsData();
-    LongColumnStatsData lcsd = new LongColumnStatsData(1, 2);
-    lcsd.setLowValue(-95);
-    lcsd.setHighValue(95);
-    csd1.setLongStats(lcsd);
-    ColumnStatisticsData csd2 = new ColumnStatisticsData();
-    csd2.setStringStats(new StringColumnStatsData(97, 18.78, 29, 397));
-    List<ColumnStatisticsObj> csos = Arrays.asList(
-        new ColumnStatisticsObj("col1", "int", csd1),
-        new ColumnStatisticsObj("col2", "varchar(32)", csd2));
-    ColumnStatistics statsObj = new ColumnStatistics(statsDesc, csos);
-    store.updatePartitionColumnStatistics(statsObj, partVals.get(1));
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.PART_TABLE, "db0.tab1.c.d", null, conf, out, err);
-    Assert.assertEquals("{\"values\":[\"c\",\"d\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
-        "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{\"COLUMN_STATS_ACCURATE\":\"{\\\"COLUMN_STATS\\\":{\\\"col1\\\":\\\"true\\\",\\\"col2\\\":\\\"true\\\"}}\"}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " +
-        "stats: column col1: {\"colName\":\"col1\",\"colType\":\"int\"," +
-        "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," +
-        "\"numDVs\":2,\"bitVectors\":\"\"}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," +
-        "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29," +
-        "\"numDVs\":397,\"bitVectors\":\"\"}}}" + lsep,  outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.PART_TABLE, null, "db0.tab1", conf, out, err);
-    Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
-        "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " +
-        "stats:" + lsep +
-        "{\"values\":[\"c\",\"d\"],\"dbName\":\"db0\",\"tableName\":\"tab1\",\"createTime\":0," +
-        "\"lastAccessTime\":0,\"parameters\":{\"COLUMN_STATS_ACCURATE\":\"{\\\"COLUMN_STATS\\\":{\\\"col1\\\":\\\"true\\\",\\\"col2\\\":\\\"true\\\"}}\"}} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column " +
-        "col1: {\"colName\":\"col1\",\"colType\":\"int\"," +
-        "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," +
-        "\"numDVs\":2,\"bitVectors\":\"\"}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," +
-        "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29," +
-        "\"numDVs\":397,\"bitVectors\":\"\"}}}" + lsep, outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.PART_TABLE, null, "db0.tab1.a", conf, out, err);
-    Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
-        "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " +
-        "stats:" + lsep, outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.SD_TABLE, "qQTgZAi5VzgpozzFGmIVTQ", null, conf, out, err);
-    Assert.assertEquals("{\"cols\":[{\"name\":\"col1\",\"type\":\"int\",\"comment\":\"\"}," +
-        "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0," +
-        "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}" + lsep,
-        outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.SD_TABLE, null, "whatever", conf, out, err);
-    Assert.assertEquals("qQTgZAi5VzgpozzFGmIVTQ: {\"cols\":[{\"name\":\"col1\",\"type\":\"int\"," +
-            "\"comment\":\"\"}," +
-            "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0," +
-            "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}" + lsep,
-        outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.SECURITY_TABLE, null, "whatever", conf, out, err);
-    Assert.assertEquals("No security related entries" + lsep, outStr.toString());
-
-    store.addMasterKey("this be a key");
-    store.addToken("tokenid", "delegation token");
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.SECURITY_TABLE, null, "whatever", conf, out, err);
-    Assert.assertEquals("Master key 0: this be a key" + lsep +
-        "Delegation token tokenid: delegation token" + lsep, outStr.toString());
-
-    outStr = new ByteArrayOutputStream();
-    out = new PrintStream(outStr);
-    tool.go(false, HBaseReadWrite.SEQUENCES_TABLE, null, "whatever", conf, out, err);
-    Assert.assertEquals("master_key: 1" + lsep, outStr.toString());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java
deleted file mode 100644
index 0c95b2f..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.commons.lang.StringUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-
-/**
- * This is in a separate class because install tests shouldn't set up the metastore first.
- */
-public class TestHBaseSchemaTool2 extends HBaseIntegrationTests {
-
-  private String lsep = System.getProperty("line.separator");
-
-  @BeforeClass
-  public static void startup() throws Exception {
-    HBaseIntegrationTests.startMiniCluster();
-  }
-
-  @AfterClass
-  public static void shutdown() throws Exception {
-    HBaseIntegrationTests.shutdownMiniCluster();
-  }
-
-  @Test
-  public void install() {
-    ByteArrayOutputStream outStr = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(outStr);
-    ByteArrayOutputStream errStr = new ByteArrayOutputStream();
-    PrintStream err = new PrintStream(errStr);
-
-    HBaseSchemaTool tool = new HBaseSchemaTool();
-    tool.install(conf, err);
-    tool.go(true, null, null, null, conf, out, err);
-    Assert.assertEquals(StringUtils.join(HBaseReadWrite.tableNames, lsep) + lsep,
-        outStr.toString());
-  }
-}


[03/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java
deleted file mode 100644
index 06884b3..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java
+++ /dev/null
@@ -1,483 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hive.metastore.PartFilterExprUtil;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.FilterPlan;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.MultiScanPlan;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan.ScanMarker;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.primitives.Shorts;
-
-public class TestHBaseFilterPlanUtil {
-  final boolean INCLUSIVE = true;
-
-  /**
-   * Test the function that compares byte arrays
-   */
-  @Test
-  public void testCompare() {
-
-    Assert.assertEquals(-1, HBaseFilterPlanUtil.compare(new byte[] { 1, 2 }, new byte[] { 1, 3 }));
-    Assert.assertEquals(-1,
-        HBaseFilterPlanUtil.compare(new byte[] { 1, 2, 3 }, new byte[] { 1, 3 }));
-    Assert.assertEquals(-1,
-        HBaseFilterPlanUtil.compare(new byte[] { 1, 2 }, new byte[] { 1, 2, 3 }));
-
-    Assert.assertEquals(0, HBaseFilterPlanUtil.compare(new byte[] { 3, 2 }, new byte[] { 3, 2 }));
-
-    Assert
-        .assertEquals(1, HBaseFilterPlanUtil.compare(new byte[] { 3, 2, 1 }, new byte[] { 3, 2 }));
-    Assert
-        .assertEquals(1, HBaseFilterPlanUtil.compare(new byte[] { 3, 3, 1 }, new byte[] { 3, 2 }));
-
-  }
-
-  /**
-   * Test function that finds greater/lesser marker
-   */
-  @Test
-  public void testgetComparedMarker() {
-    ScanMarker l;
-    ScanMarker r;
-
-    // equal plans
-    l = new ScanMarker("1", INCLUSIVE, "int");
-    r = new ScanMarker("1", INCLUSIVE, "int");
-    assertFirstGreater(l, r);
-
-    l = new ScanMarker("1", !INCLUSIVE, "int");
-    r = new ScanMarker("1", !INCLUSIVE, "int");
-    assertFirstGreater(l, r);
-
-    assertFirstGreater(null, null);
-
-    // create l is greater because of inclusive flag
-    l = new ScanMarker("1", !INCLUSIVE, "int");
-    // the rule for null vs non-null is different
-    // non-null is both smaller and greater than null
-    Assert.assertEquals(l, ScanPlan.getComparedMarker(l, null, true));
-    Assert.assertEquals(l, ScanPlan.getComparedMarker(null, l, true));
-    Assert.assertEquals(l, ScanPlan.getComparedMarker(l, null, false));
-    Assert.assertEquals(l, ScanPlan.getComparedMarker(null, l, false));
-
-    // create l that is greater because of the bytes
-    l = new ScanMarker("2", INCLUSIVE, "int");
-    r = new ScanMarker("1", INCLUSIVE, "int");
-    assertFirstGreater(l, r);
-
-  }
-
-  private void assertFirstGreater(ScanMarker big, ScanMarker small) {
-    Assert.assertEquals(big, ScanPlan.getComparedMarker(big, small, true));
-    Assert.assertEquals(big, ScanPlan.getComparedMarker(small, big, true));
-    Assert.assertEquals(small, ScanPlan.getComparedMarker(big, small, false));
-    Assert.assertEquals(small, ScanPlan.getComparedMarker(small, big, false));
-  }
-
-  /**
-   * Test ScanPlan AND operation
-   */
-  @Test
-  public void testScanPlanAnd() {
-    ScanPlan l = new ScanPlan();
-    ScanPlan r = new ScanPlan();
-    l.setStartMarker("a", "int", "10", INCLUSIVE);
-    r.setStartMarker("a", "int", "10", INCLUSIVE);
-
-    ScanPlan res;
-    // both equal
-    res = l.and(r).getPlans().get(0);
-    Assert.assertEquals(new ScanMarker("10", INCLUSIVE, "int"), res.markers.get("a").startMarker);
-
-    // add equal end markers as well, and test AND again
-    l.setEndMarker("a", "int", "20", INCLUSIVE);
-    r.setEndMarker("a", "int", "20", INCLUSIVE);
-    res = l.and(r).getPlans().get(0);
-    Assert.assertEquals(new ScanMarker("10", INCLUSIVE, "int"), res.markers.get("a").startMarker);
-    Assert.assertEquals(new ScanMarker("20", INCLUSIVE, "int"), res.markers.get("a").endMarker);
-
-    l.setStartMarker("a", "int", "10", !INCLUSIVE);
-    l.setEndMarker("a", "int", "20", INCLUSIVE);
-
-    r.setStartMarker("a", "int", "10", INCLUSIVE);
-    r.setEndMarker("a", "int", "15", INCLUSIVE);
-    res = l.and(r).getPlans().get(0);
-    // start of l is greater, end of r is smaller
-    Assert.assertEquals(l.markers.get("a").startMarker, res.markers.get("a").startMarker);
-    Assert.assertEquals(r.markers.get("a").endMarker, res.markers.get("a").endMarker);
-
-  }
-
-  /**
-   * Test ScanPlan OR operation
-   */
-  @Test
-  public void testScanPlanOr() {
-    ScanPlan l = new ScanPlan();
-    ScanPlan r = new ScanPlan();
-    l.setStartMarker("a", "int", "1", INCLUSIVE);
-    r.setStartMarker("a", "int", "11", INCLUSIVE);
-
-    FilterPlan res1 = l.or(r);
-    Assert.assertEquals(2, res1.getPlans().size());
-    res1.getPlans().get(0).markers.get("a").startMarker.equals(l.markers.get("a").startMarker);
-    res1.getPlans().get(1).markers.get("a").startMarker.equals(r.markers.get("a").startMarker);
-
-    FilterPlan res2 = res1.or(r);
-    Assert.assertEquals(3, res2.getPlans().size());
-  }
-
-  /**
-   * Test MultiScanPlan OR
-   */
-  @Test
-  public void testMultiScanPlanOr() {
-
-    MultiScanPlan l = createMultiScanPlan(new ScanPlan());
-    MultiScanPlan r = createMultiScanPlan(new ScanPlan());
-    // verify OR of two multi plans with one plan each
-    Assert.assertEquals(2, l.or(r).getPlans().size());
-
-    // verify OR of multi plan with a single scanplan
-    Assert.assertEquals(2, l.or(new ScanPlan()).getPlans().size());
-    Assert.assertEquals(2, (new ScanPlan()).or(l).getPlans().size());
-
-    // verify or of two multiplans with more than one scan plan
-    r = createMultiScanPlan(new ScanPlan(), new ScanPlan());
-    Assert.assertEquals(3, l.or(r).getPlans().size());
-    Assert.assertEquals(3, r.or(l).getPlans().size());
-
-  }
-
-  private MultiScanPlan createMultiScanPlan(ScanPlan... scanPlans) {
-    return new MultiScanPlan(Arrays.asList(scanPlans));
-  }
-
-  /**
-   * Test MultiScanPlan AND
-   */
-  @Test
-  public void testMultiScanPlanAnd() {
-    MultiScanPlan l = createMultiScanPlan(new ScanPlan());
-    MultiScanPlan r = createMultiScanPlan(new ScanPlan());
-
-    // two MultiScanPlan with single scan plans should result in new FilterPlan
-    // with just one scan
-    Assert.assertEquals(1, l.and(r).getPlans().size());
-
-    // l has one ScanPlan, r has two. AND result should have two
-    r = createMultiScanPlan(new ScanPlan(), new ScanPlan());
-    Assert.assertEquals(2, l.and(r).getPlans().size());
-    Assert.assertEquals(2, r.and(l).getPlans().size());
-
-    // l has 2 ScanPlans, r has 3. AND result should have 6
-    l = createMultiScanPlan(new ScanPlan(), new ScanPlan());
-    r = createMultiScanPlan(new ScanPlan(), new ScanPlan(), new ScanPlan());
-    Assert.assertEquals(6, l.and(r).getPlans().size());
-    Assert.assertEquals(6, r.and(l).getPlans().size());
-  }
-
-  /**
-   * Test plan generation from LeafNode
-   *
-   * @throws MetaException
-   */
-  @Test
-  public void testLeafNodePlan() throws MetaException {
-
-    final String KEY = "k1";
-    final String VAL = "v1";
-    final String OTHERKEY = "k2";
-    LeafNode l = new LeafNode();
-    l.keyName = KEY;
-    l.value = VAL;
-    final ScanMarker DEFAULT_SCANMARKER = null;
-    List<FieldSchema> parts = new ArrayList<FieldSchema>();
-    parts.add(new FieldSchema(KEY, "int", null));
-    parts.add(new FieldSchema(OTHERKEY, "int", null));
-
-    l.operator = Operator.EQUALS;
-    verifyPlan(l, parts, KEY, new ScanMarker(VAL, INCLUSIVE, "int"), new ScanMarker(VAL, INCLUSIVE, "int"));
-
-    l.operator = Operator.GREATERTHAN;
-    verifyPlan(l, parts, KEY, new ScanMarker(VAL, !INCLUSIVE, "int"), DEFAULT_SCANMARKER);
-
-    l.operator = Operator.GREATERTHANOREQUALTO;
-    verifyPlan(l, parts, KEY, new ScanMarker(VAL, INCLUSIVE, "int"), DEFAULT_SCANMARKER);
-
-    l.operator = Operator.LESSTHAN;
-    verifyPlan(l, parts, KEY, DEFAULT_SCANMARKER, new ScanMarker(VAL, !INCLUSIVE, "int"));
-
-    l.operator = Operator.LESSTHANOREQUALTO;
-    verifyPlan(l, parts, KEY, DEFAULT_SCANMARKER, new ScanMarker(VAL, INCLUSIVE, "int"));
-
-    // following leaf node plans should currently have true for 'has unsupported condition',
-    // because of the condition is not on first key
-    l.operator = Operator.EQUALS;
-    verifyPlan(l, parts, OTHERKEY, DEFAULT_SCANMARKER, DEFAULT_SCANMARKER, false);
-
-    // if tree is null, it should return equivalent of full scan, and true
-    // for 'has unsupported condition'
-    verifyPlan(null, parts, KEY, DEFAULT_SCANMARKER, DEFAULT_SCANMARKER, true);
-
-  }
-
-  private void verifyPlan(TreeNode l, List<FieldSchema> parts, String keyName, ScanMarker startMarker, ScanMarker endMarker)
-      throws MetaException {
-    verifyPlan(l, parts, keyName, startMarker, endMarker, false);
-  }
-
-  private void verifyPlan(TreeNode l, List<FieldSchema> parts, String keyName, ScanMarker startMarker, ScanMarker endMarker,
-      boolean hasUnsupportedCondition) throws MetaException {
-    ExpressionTree e = null;
-    if (l != null) {
-      e = new ExpressionTree();
-      e.setRootForTest(l);
-    }
-    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
-    FilterPlan plan = planRes.plan;
-    Assert.assertEquals("Has unsupported condition", hasUnsupportedCondition,
-        planRes.hasUnsupportedCondition);
-    Assert.assertEquals(1, plan.getPlans().size());
-    ScanPlan splan = plan.getPlans().get(0);
-    if (startMarker != null) {
-      Assert.assertEquals(startMarker, splan.markers.get(keyName).startMarker);
-    } else {
-      Assert.assertTrue(splan.markers.get(keyName)==null ||
-          splan.markers.get(keyName).startMarker==null);
-    }
-    if (endMarker != null) {
-      Assert.assertEquals(endMarker, splan.markers.get(keyName).endMarker);
-    } else {
-      Assert.assertTrue(splan.markers.get(keyName)==null ||
-          splan.markers.get(keyName).endMarker==null);
-    }
-  }
-
-  /**
-   * Test plan generation from TreeNode
-   *
-   * @throws MetaException
-   */
-  @Test
-  public void testTreeNodePlan() throws MetaException {
-
-    final String KEY = "k1";
-    final String VAL1 = "10";
-    final String VAL2 = "11";
-    LeafNode l = new LeafNode();
-    l.keyName = KEY;
-    l.value = VAL1;
-    final ScanMarker DEFAULT_SCANMARKER = null;
-
-    List<FieldSchema> parts = new ArrayList<FieldSchema>();
-    parts.add(new FieldSchema("k1", "int", null));
-
-    LeafNode r = new LeafNode();
-    r.keyName = KEY;
-    r.value = VAL2;
-
-    TreeNode tn = new TreeNode(l, LogicalOperator.AND, r);
-
-    // verify plan for - k1 >= '10' and k1 < '11'
-    l.operator = Operator.GREATERTHANOREQUALTO;
-    r.operator = Operator.LESSTHAN;
-    verifyPlan(tn, parts, KEY, new ScanMarker(VAL1, INCLUSIVE, "int"), new ScanMarker(VAL2,
-        !INCLUSIVE, "int"));
-
-    // verify plan for - k1 >= '10' and k1 > '11'
-    l.operator = Operator.GREATERTHANOREQUALTO;
-    r.operator = Operator.GREATERTHAN;
-    verifyPlan(tn, parts, KEY, new ScanMarker(VAL2, !INCLUSIVE, "int"), DEFAULT_SCANMARKER);
-
-    // verify plan for - k1 >= '10' or k1 > '11'
-    tn = new TreeNode(l, LogicalOperator.OR, r);
-    ExpressionTree e = new ExpressionTree();
-    e.setRootForTest(tn);
-    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
-    Assert.assertEquals(2, planRes.plan.getPlans().size());
-    Assert.assertEquals(false, planRes.hasUnsupportedCondition);
-
-    // verify plan for - k1 >= '10' and (k1 >= '10' or k1 > '11')
-    TreeNode tn2 = new TreeNode(l, LogicalOperator.AND, tn);
-    e = new ExpressionTree();
-    e.setRootForTest(tn2);
-    planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
-    Assert.assertEquals(2, planRes.plan.getPlans().size());
-    Assert.assertEquals(false, planRes.hasUnsupportedCondition);
-
-    // verify plan for  (k1 >= '10' and (k1 >= '10' or k1 > '11')) or k1 LIKE '2'
-    // plan should return true for hasUnsupportedCondition
-    LeafNode klike = new LeafNode();
-    klike.keyName = KEY;
-    klike.value = VAL1;
-    klike.operator = Operator.LIKE;
-    TreeNode tn3 = new TreeNode(tn2, LogicalOperator.OR, klike);
-    e = new ExpressionTree();
-    e.setRootForTest(tn3);
-    planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
-    Assert.assertEquals(3, planRes.plan.getPlans().size());
-    Assert.assertEquals(false, planRes.hasUnsupportedCondition);
-
-
-  }
-
-  @Test
-  public void testPartitionKeyScannerAllString() throws Exception {
-    List<FieldSchema> parts = new ArrayList<FieldSchema>();
-    parts.add(new FieldSchema("year", "string", null));
-    parts.add(new FieldSchema("month", "string", null));
-    parts.add(new FieldSchema("state", "string", null));
-
-    // One prefix key and one minor key range
-    ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
-    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
-    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
-    ScanPlan sp = planRes.plan.getPlans().get(0);
-    byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
-    byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
-    RowFilter filter = (RowFilter)sp.getFilter(parts);
-
-    // scan range contains the major key year, rowfilter contains minor key state
-    Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
-    Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
-    Assert.assertFalse(Bytes.contains(startRowSuffix, "CA".getBytes()));
-    Assert.assertFalse(Bytes.contains(endRowSuffix, "CA".getBytes()));
-
-    PartitionKeyComparator comparator = (PartitionKeyComparator)filter.getComparator();
-    Assert.assertEquals(comparator.ranges.size(), 1);
-    Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
-
-    // Two prefix key and one LIKE operator
-    exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and month > 10 "
-        + "and month <= 11 and state like 'C%'").tree;
-    planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
-    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
-    sp = planRes.plan.getPlans().get(0);
-    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
-    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
-    filter = (RowFilter)sp.getFilter(parts);
-
-    // scan range contains the major key value year/month, rowfilter contains LIKE operator
-    Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
-    Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
-    Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
-    Assert.assertTrue(Bytes.contains(endRowSuffix, "11".getBytes()));
-
-    comparator = (PartitionKeyComparator)filter.getComparator();
-    Assert.assertEquals(comparator.ops.size(), 1);
-    Assert.assertEquals(comparator.ops.get(0).keyName, "state");
-
-    // One prefix key, one minor key range and one LIKE operator
-    exprTree = PartFilterExprUtil.getFilterParser("year >= 2014 and month > 10 "
-        + "and month <= 11 and state like 'C%'").tree;
-    planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
-    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
-    sp = planRes.plan.getPlans().get(0);
-    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
-    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
-    filter = (RowFilter)sp.getFilter(parts);
-
-    // scan range contains the major key value year (low bound), rowfilter contains minor key state
-    // and LIKE operator
-    Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
-
-    comparator = (PartitionKeyComparator)filter.getComparator();
-    Assert.assertEquals(comparator.ranges.size(), 1);
-    Assert.assertEquals(comparator.ranges.get(0).keyName, "month");
-    Assert.assertEquals(comparator.ops.size(), 1);
-    Assert.assertEquals(comparator.ops.get(0).keyName, "state");
-
-    // Condition contains or
-    exprTree = PartFilterExprUtil.getFilterParser("year = 2014 and (month > 10 "
-        + "or month < 3)").tree;
-    planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
-    sp = planRes.plan.getPlans().get(0);
-    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
-    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
-    filter = (RowFilter)sp.getFilter(parts);
-
-    // The first ScanPlan contains year = 2014 and month > 10
-    Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
-    Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
-    Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
-
-    sp = planRes.plan.getPlans().get(1);
-    startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
-    endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
-    filter = (RowFilter)sp.getFilter(parts);
-
-    // The first ScanPlan contains year = 2014 and month < 3
-    Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
-    Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
-    Assert.assertTrue(Bytes.contains(endRowSuffix, "3".getBytes()));
-  }
-
-  @Test
-  public void testPartitionKeyScannerMixedType() throws Exception {
-    List<FieldSchema> parts = new ArrayList<FieldSchema>();
-    parts.add(new FieldSchema("year", "int", null));
-    parts.add(new FieldSchema("month", "int", null));
-    parts.add(new FieldSchema("state", "string", null));
-
-    // One prefix key and one minor key range
-    ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
-    PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
-    Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
-    ScanPlan sp = planRes.plan.getPlans().get(0);
-    byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
-    byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
-    RowFilter filter = (RowFilter)sp.getFilter(parts);
-
-    // scan range contains the major key year, rowfilter contains minor key state
-    Assert.assertTrue(Bytes.contains(startRowSuffix, Shorts.toByteArray((short)2015)));
-    Assert.assertTrue(Bytes.contains(endRowSuffix, Shorts.toByteArray((short)2016)));
-
-    PartitionKeyComparator comparator = (PartitionKeyComparator)filter.getComparator();
-    Assert.assertEquals(comparator.ranges.size(), 1);
-    Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
-  }
-}


[02/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
deleted file mode 100644
index 4aa8c34..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
+++ /dev/null
@@ -1,1978 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.ResourceType;
-import org.apache.hadoop.hive.metastore.api.ResourceUri;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class TestHBaseStore {
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStore.class.getName());
-  static Map<String, String> emptyParameters = new HashMap<String, String>();
-  // Table with NUM_PART_KEYS partitioning keys and NUM_PARTITIONS values per key
-  static final int NUM_PART_KEYS = 1;
-  static final int NUM_PARTITIONS = 5;
-  static final String DB = "db";
-  static final String TBL = "tbl";
-  static final String COL = "col";
-  static final String PART_KEY_PREFIX = "part";
-  static final String PART_VAL_PREFIX = "val";
-  static final String PART_KV_SEPARATOR = "=";
-  static final List<String> PART_KEYS = new ArrayList<String>();
-  static final List<String> PART_VALS = new ArrayList<String>();
-  // Initialize mock partitions
-  static {
-    for (int i = 1; i <= NUM_PART_KEYS; i++) {
-      PART_KEYS.add(PART_KEY_PREFIX + i);
-    }
-    for (int i = 1; i <= NUM_PARTITIONS; i++) {
-      PART_VALS.add(PART_VAL_PREFIX + i);
-    }
-  }
-  static final long DEFAULT_TIME = System.currentTimeMillis();
-  static final String PART_KEY = "part";
-  static final String BOOLEAN_COL = "boolCol";
-  static final String BOOLEAN_TYPE = "boolean";
-  static final String BOOLEAN_VAL = "true";
-  static final String LONG_COL = "longCol";
-  static final String LONG_TYPE = "long";
-  static final String INT_TYPE = "int";
-  static final String INT_VAL = "1234";
-  static final String DOUBLE_COL = "doubleCol";
-  static final String DOUBLE_TYPE = "double";
-  static final String DOUBLE_VAL = "3.1415";
-  static final String STRING_COL = "stringCol";
-  static final String STRING_TYPE = "string";
-  static final String STRING_VAL = "stringval";
-  static final String BINARY_COL = "binaryCol";
-  static final String BINARY_TYPE = "binary";
-  static final String BINARY_VAL = "1";
-  static final String DECIMAL_COL = "decimalCol";
-  static final String DECIMAL_TYPE = "decimal(5,3)";
-  static final String DECIMAL_VAL = "12.123";
-  static List<ColumnStatisticsObj> booleanColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-  static List<ColumnStatisticsObj> longColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-  static List<ColumnStatisticsObj> doubleColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-  static List<ColumnStatisticsObj> stringColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-  static List<ColumnStatisticsObj> binaryColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-  static List<ColumnStatisticsObj> decimalColStatsObjs = new ArrayList<ColumnStatisticsObj>(
-      NUM_PARTITIONS);
-
-  @Rule public ExpectedException thrown = ExpectedException.none();
-  @Mock HTableInterface htable;
-  SortedMap<String, Cell> rows = new TreeMap<>();
-  HBaseStore store;
-
-
-  @BeforeClass
-  public static void beforeTest() {
-    // All data intitializations
-    populateMockStats();
-  }
-
-  private static void populateMockStats() {
-    ColumnStatisticsObj statsObj;
-    // Add NUM_PARTITIONS ColumnStatisticsObj of each type
-    // For aggregate stats test, we'll treat each ColumnStatisticsObj as stats for 1 partition
-    // For the rest, we'll just pick the 1st ColumnStatisticsObj from this list and use it
-    for (int i = 0; i < NUM_PARTITIONS; i++) {
-      statsObj = mockBooleanStats(i);
-      booleanColStatsObjs.add(statsObj);
-      statsObj = mockLongStats(i);
-      longColStatsObjs.add(statsObj);
-      statsObj = mockDoubleStats(i);
-      doubleColStatsObjs.add(statsObj);
-      statsObj = mockStringStats(i);
-      stringColStatsObjs.add(statsObj);
-      statsObj = mockBinaryStats(i);
-      binaryColStatsObjs.add(statsObj);
-      statsObj = mockDecimalStats(i);
-      decimalColStatsObjs.add(statsObj);
-    }
-  }
-
-  private static ColumnStatisticsObj mockBooleanStats(int i) {
-    long trues = 37 + 100*i;
-    long falses = 12 + 50*i;
-    long nulls = 2 + i;
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(BOOLEAN_COL);
-    colStatsObj.setColType(BOOLEAN_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    BooleanColumnStatsData boolData = new BooleanColumnStatsData();
-    boolData.setNumTrues(trues);
-    boolData.setNumFalses(falses);
-    boolData.setNumNulls(nulls);
-    data.setBooleanStats(boolData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  private static ColumnStatisticsObj mockLongStats(int i) {
-    long high = 120938479124L + 100*i;
-    long low = -12341243213412124L - 50*i;
-    long nulls = 23 + i;
-    long dVs = 213L + 10*i;
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(LONG_COL);
-    colStatsObj.setColType(LONG_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    LongColumnStatsData longData = new LongColumnStatsData();
-    longData.setHighValue(high);
-    longData.setLowValue(low);
-    longData.setNumNulls(nulls);
-    longData.setNumDVs(dVs);
-    data.setLongStats(longData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  private static ColumnStatisticsObj mockDoubleStats(int i) {
-    double high = 123423.23423 + 100*i;
-    double low = 0.00001234233 - 50*i;
-    long nulls = 92 + i;
-    long dVs = 1234123421L + 10*i;
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(DOUBLE_COL);
-    colStatsObj.setColType(DOUBLE_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    DoubleColumnStatsData doubleData = new DoubleColumnStatsData();
-    doubleData.setHighValue(high);
-    doubleData.setLowValue(low);
-    doubleData.setNumNulls(nulls);
-    doubleData.setNumDVs(dVs);
-    data.setDoubleStats(doubleData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  private static ColumnStatisticsObj mockStringStats(int i) {
-    long maxLen = 1234 + 10*i;
-    double avgLen = 32.3 + i;
-    long nulls = 987 + 10*i;
-    long dVs = 906 + i;
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(STRING_COL);
-    colStatsObj.setColType(STRING_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    StringColumnStatsData stringData = new StringColumnStatsData();
-    stringData.setMaxColLen(maxLen);
-    stringData.setAvgColLen(avgLen);
-    stringData.setNumNulls(nulls);
-    stringData.setNumDVs(dVs);
-    data.setStringStats(stringData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  private static ColumnStatisticsObj mockBinaryStats(int i) {;
-    long maxLen = 123412987L + 10*i;
-    double avgLen = 76.98 + i;
-    long nulls = 976998797L + 10*i;
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(BINARY_COL);
-    colStatsObj.setColType(BINARY_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    BinaryColumnStatsData binaryData = new BinaryColumnStatsData();
-    binaryData.setMaxColLen(maxLen);
-    binaryData.setAvgColLen(avgLen);
-    binaryData.setNumNulls(nulls);
-    data.setBinaryStats(binaryData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  private static ColumnStatisticsObj mockDecimalStats(int i) {
-    Decimal high = new Decimal();
-    high.setScale((short)3);
-    String strHigh = String.valueOf(3876 + 100*i);
-    high.setUnscaled(strHigh.getBytes());
-    Decimal low = new Decimal();
-    low.setScale((short)3);
-    String strLow = String.valueOf(38 + i);
-    low.setUnscaled(strLow.getBytes());
-    long nulls = 13 + i;
-    long dVs = 923947293L + 100*i;
-    ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
-    colStatsObj.setColName(DECIMAL_COL);
-    colStatsObj.setColType(DECIMAL_TYPE);
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    DecimalColumnStatsData decimalData = new DecimalColumnStatsData();
-    decimalData.setHighValue(high);
-    decimalData.setLowValue(low);
-    decimalData.setNumNulls(nulls);
-    decimalData.setNumDVs(dVs);
-    data.setDecimalStats(decimalData);
-    colStatsObj.setStatsData(data);
-    return colStatsObj;
-  }
-
-  @AfterClass
-  public static void afterTest() {
-  }
-
-
-  @Before
-  public void init() throws IOException {
-    MockitoAnnotations.initMocks(this);
-    HiveConf conf = new HiveConf();
-    conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
-    store = MockUtils.init(conf, htable, rows);
-  }
-
-  @Test
-  public void createDb() throws Exception {
-    String dbname = "mydb";
-    Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
-    store.createDatabase(db);
-
-    Database d = store.getDatabase(dbname);
-    Assert.assertEquals(dbname, d.getName());
-    Assert.assertEquals("no description", d.getDescription());
-    Assert.assertEquals("file:///tmp", d.getLocationUri());
-  }
-
-  @Test
-  public void alterDb() throws Exception {
-    String dbname = "mydb";
-    Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
-    store.createDatabase(db);
-    db.setDescription("a description");
-    store.alterDatabase(dbname, db);
-
-    Database d = store.getDatabase(dbname);
-    Assert.assertEquals(dbname, d.getName());
-    Assert.assertEquals("a description", d.getDescription());
-    Assert.assertEquals("file:///tmp", d.getLocationUri());
-  }
-
-  @Test
-  public void dropDb() throws Exception {
-    String dbname = "anotherdb";
-    Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
-    store.createDatabase(db);
-
-    Database d = store.getDatabase(dbname);
-    Assert.assertNotNull(d);
-
-    store.dropDatabase(dbname);
-    thrown.expect(NoSuchObjectException.class);
-    store.getDatabase(dbname);
-  }
-
-  @Test
-  public void createFunction() throws Exception {
-    String funcName = "createfunc";
-    int now = (int)(System.currentTimeMillis()/ 1000);
-    Function func = new Function(funcName, DB, "o.a.h.h.myfunc", "me", PrincipalType.USER,
-        now, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR,
-        "file:/tmp/somewhere")));
-    store.createFunction(func);
-
-    Function f = store.getFunction(DB, funcName);
-    Assert.assertEquals(DB, f.getDbName());
-    Assert.assertEquals(funcName, f.getFunctionName());
-    Assert.assertEquals("o.a.h.h.myfunc", f.getClassName());
-    Assert.assertEquals("me", f.getOwnerName());
-    Assert.assertEquals(PrincipalType.USER, f.getOwnerType());
-    Assert.assertTrue(now <= f.getCreateTime());
-    Assert.assertEquals(FunctionType.JAVA, f.getFunctionType());
-    Assert.assertEquals(1, f.getResourceUrisSize());
-    Assert.assertEquals(ResourceType.JAR, f.getResourceUris().get(0).getResourceType());
-    Assert.assertEquals("file:/tmp/somewhere", f.getResourceUris().get(0).getUri());
-  }
-
-  @Test
-  public void alterFunction() throws Exception {
-    String funcName = "alterfunc";
-    int now = (int)(System.currentTimeMillis()/ 1000);
-    List<ResourceUri> uris = new ArrayList<ResourceUri>();
-    uris.add(new ResourceUri(ResourceType.FILE, "whatever"));
-    Function func = new Function(funcName, DB, "o.a.h.h.myfunc", "me", PrincipalType.USER,
-        now, FunctionType.JAVA, uris);
-    store.createFunction(func);
-
-    Function f = store.getFunction(DB, funcName);
-    Assert.assertEquals(ResourceType.FILE, f.getResourceUris().get(0).getResourceType());
-
-    func.addToResourceUris(new ResourceUri(ResourceType.ARCHIVE, "file"));
-    store.alterFunction(DB, funcName, func);
-
-    f = store.getFunction(DB, funcName);
-    Assert.assertEquals(2, f.getResourceUrisSize());
-    Assert.assertEquals(ResourceType.FILE, f.getResourceUris().get(0).getResourceType());
-    Assert.assertEquals(ResourceType.ARCHIVE, f.getResourceUris().get(1).getResourceType());
-
-  }
-
-  @Test
-  public void dropFunction() throws Exception {
-    String funcName = "delfunc";
-    int now = (int)(System.currentTimeMillis()/ 1000);
-    Function func = new Function(funcName, DB, "o.a.h.h.myfunc", "me", PrincipalType.USER,
-        now, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR, "file:/tmp/somewhere")));
-    store.createFunction(func);
-
-    Function f = store.getFunction(DB, funcName);
-    Assert.assertNotNull(f);
-
-    store.dropFunction(DB, funcName);
-    //thrown.expect(NoSuchObjectException.class);
-    Assert.assertNull(store.getFunction(DB, funcName));
-  }
-
-  @Test
-  public void createTable() throws Exception {
-    String tableName = "mytable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", ""));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    Map<String, String> params = new HashMap<String, String>();
-    params.put("key", "value");
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
-        serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    Table t = store.getTable("default", tableName);
-    Assert.assertEquals(1, t.getSd().getColsSize());
-    Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
-    Assert.assertEquals("", t.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp", t.getSd().getLocation());
-    Assert.assertEquals("input", t.getSd().getInputFormat());
-    Assert.assertEquals("output", t.getSd().getOutputFormat());
-    Assert.assertFalse(t.getSd().isCompressed());
-    Assert.assertEquals(17, t.getSd().getNumBuckets());
-    Assert.assertEquals(1, t.getSd().getBucketColsSize());
-    Assert.assertEquals("bucketcol", t.getSd().getBucketCols().get(0));
-    Assert.assertEquals(1, t.getSd().getSortColsSize());
-    Assert.assertEquals("sortcol", t.getSd().getSortCols().get(0).getCol());
-    Assert.assertEquals(1, t.getSd().getSortCols().get(0).getOrder());
-    Assert.assertEquals(1, t.getSd().getParametersSize());
-    Assert.assertEquals("value", t.getSd().getParameters().get("key"));
-    Assert.assertEquals("me", t.getOwner());
-    Assert.assertEquals("default", t.getDbName());
-    Assert.assertEquals(tableName, t.getTableName());
-    Assert.assertEquals(0, t.getParametersSize());
-  }
-
-  @Test
-  public void skewInfo() throws Exception {
-    String tableName = "mytable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", ""));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", true, 0,
-        serde, null, null, emptyParameters);
-
-    Map<List<String>, String> map = new HashMap<List<String>, String>();
-    map.put(Arrays.asList("col3"), "col4");
-    SkewedInfo skew = new SkewedInfo(Arrays.asList("col1"), Arrays.asList(Arrays.asList("col2")),
-        map);
-    sd.setSkewedInfo(skew);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    Table t = store.getTable("default", tableName);
-    Assert.assertEquals(1, t.getSd().getColsSize());
-    Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
-    Assert.assertEquals("", t.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp", t.getSd().getLocation());
-    Assert.assertEquals("input", t.getSd().getInputFormat());
-    Assert.assertEquals("output", t.getSd().getOutputFormat());
-    Assert.assertTrue(t.getSd().isCompressed());
-    Assert.assertEquals(0, t.getSd().getNumBuckets());
-    Assert.assertEquals(0, t.getSd().getSortColsSize());
-    Assert.assertEquals("me", t.getOwner());
-    Assert.assertEquals("default", t.getDbName());
-    Assert.assertEquals(tableName, t.getTableName());
-    Assert.assertEquals(0, t.getParametersSize());
-
-    skew = t.getSd().getSkewedInfo();
-    Assert.assertNotNull(skew);
-    Assert.assertEquals(1, skew.getSkewedColNamesSize());
-    Assert.assertEquals("col1", skew.getSkewedColNames().get(0));
-    Assert.assertEquals(1, skew.getSkewedColValuesSize());
-    Assert.assertEquals("col2", skew.getSkewedColValues().get(0).get(0));
-    Assert.assertEquals(1, skew.getSkewedColValueLocationMapsSize());
-    Assert.assertEquals("col4", skew.getSkewedColValueLocationMaps().get(Arrays.asList("col3")));
-
-  }
-
-  @Test
-  public void hashSd() throws Exception {
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", ""));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", true, 0,
-        serde, null, null, emptyParameters);
-
-    Map<List<String>, String> map = new HashMap<List<String>, String>();
-    map.put(Arrays.asList("col3"), "col4");
-    SkewedInfo skew = new SkewedInfo(Arrays.asList("col1"), Arrays.asList(Arrays.asList("col2")),
-        map);
-    sd.setSkewedInfo(skew);
-
-    MessageDigest md = MessageDigest.getInstance("MD5");
-    byte[] baseHash = HBaseUtils.hashStorageDescriptor(sd, md);
-
-    StorageDescriptor changeSchema = new StorageDescriptor(sd);
-    changeSchema.getCols().add(new FieldSchema("col2", "varchar(32)", "a comment"));
-    byte[] schemaHash = HBaseUtils.hashStorageDescriptor(changeSchema, md);
-    Assert.assertFalse(Arrays.equals(baseHash, schemaHash));
-
-    StorageDescriptor changeLocation = new StorageDescriptor(sd);
-    changeLocation.setLocation("file:/somewhere/else");
-    byte[] locationHash = HBaseUtils.hashStorageDescriptor(changeLocation, md);
-    Assert.assertArrayEquals(baseHash, locationHash);
-  }
-
-  @Test
-  public void alterTable() throws Exception {
-    String tableName = "alttable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    startTime += 10;
-    table.setLastAccessTime(startTime);
-    store.alterTable("default", tableName, table);
-
-    Table t = store.getTable("default", tableName);
-    Assert.assertEquals(1, t.getSd().getColsSize());
-    Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
-    Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp", t.getSd().getLocation());
-    Assert.assertEquals("input", t.getSd().getInputFormat());
-    Assert.assertEquals("output", t.getSd().getOutputFormat());
-    Assert.assertEquals("me", t.getOwner());
-    Assert.assertEquals("default", t.getDbName());
-    Assert.assertEquals(tableName, t.getTableName());
-    Assert.assertEquals(startTime, t.getLastAccessTime());
-  }
-
-  @Test
-  public void dropTable() throws Exception {
-    String tableName = "dtable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    Table t = store.getTable("default", tableName);
-    Assert.assertNotNull(t);
-
-    store.dropTable("default", tableName);
-    Assert.assertNull(store.getTable("default", tableName));
-  }
-
-  @Test
-  public void createPartition() throws Exception {
-    String tableName = "myparttable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> vals = Arrays.asList("fred");
-    StorageDescriptor psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/pc=fred");
-    Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
-        emptyParameters);
-    store.addPartition(part);
-
-    Partition p = store.getPartition(DB, tableName, vals);
-    Assert.assertEquals(1, p.getSd().getColsSize());
-    Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
-    Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
-    Assert.assertEquals("input", p.getSd().getInputFormat());
-    Assert.assertEquals("output", p.getSd().getOutputFormat());
-    Assert.assertEquals(DB, p.getDbName());
-    Assert.assertEquals(tableName, p.getTableName());
-    Assert.assertEquals(1, p.getValuesSize());
-    Assert.assertEquals("fred", p.getValues().get(0));
-
-    Assert.assertTrue(store.doesPartitionExist(DB, tableName, vals));
-    Assert.assertFalse(store.doesPartitionExist(DB, tableName, Arrays.asList("bob")));
-  }
-
-  @Test
-  public void alterPartition() throws Exception {
-    String tableName = "alterparttable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> vals = Arrays.asList("fred");
-    StorageDescriptor psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/pc=fred");
-    Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
-        emptyParameters);
-    store.addPartition(part);
-
-    part.setLastAccessTime(startTime + 10);
-    store.alterPartition(DB, tableName, vals, part);
-
-    Partition p = store.getPartition(DB, tableName, vals);
-    Assert.assertEquals(1, p.getSd().getColsSize());
-    Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
-    Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
-    Assert.assertEquals("input", p.getSd().getInputFormat());
-    Assert.assertEquals("output", p.getSd().getOutputFormat());
-    Assert.assertEquals(DB, p.getDbName());
-    Assert.assertEquals(tableName, p.getTableName());
-    Assert.assertEquals(1, p.getValuesSize());
-    Assert.assertEquals("fred", p.getValues().get(0));
-    Assert.assertEquals(startTime + 10, p.getLastAccessTime());
-
-    Assert.assertTrue(store.doesPartitionExist(DB, tableName, vals));
-    Assert.assertFalse(store.doesPartitionExist(DB, tableName, Arrays.asList("bob")));
-  }
-
-  @Test
-  public void getPartitions() throws Exception {
-    String tableName = "manyParts";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
-    for (String val : partVals) {
-      List<String> vals = new ArrayList<String>();
-      vals.add(val);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/pc=" + val);
-      Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
-          emptyParameters);
-      store.addPartition(part);
-
-      Partition p = store.getPartition(DB, tableName, vals);
-      Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
-    }
-
-    List<Partition> parts = store.getPartitions(DB, tableName, -1);
-    Assert.assertEquals(5, parts.size());
-    String[] pv = new String[5];
-    for (int i = 0; i < 5; i++) pv[i] = parts.get(i).getValues().get(0);
-    Arrays.sort(pv);
-    Assert.assertArrayEquals(pv, partVals.toArray(new String[5]));
-  }
-
-  @Test
-  public void listGetDropPartitionNames() throws Exception {
-    String tableName = "listParts";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    partCols.add(new FieldSchema("region", "string", ""));
-    Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    String[][] partVals = new String[][]{{"today", "north america"}, {"tomorrow", "europe"}};
-    for (String[] pv : partVals) {
-      List<String> vals = new ArrayList<String>();
-      for (String v : pv) vals.add(v);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/pc=" + pv[0] + "/region=" + pv[1]);
-      Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
-          emptyParameters);
-      store.addPartition(part);
-    }
-
-    List<String> names = store.listPartitionNames(DB, tableName, (short) -1);
-    Assert.assertEquals(2, names.size());
-    String[] resultNames = names.toArray(new String[names.size()]);
-    Arrays.sort(resultNames);
-    Assert.assertArrayEquals(resultNames, new String[]{"pc=today/region=north america",
-      "pc=tomorrow/region=europe"});
-
-    List<Partition> parts = store.getPartitionsByNames(DB, tableName, names);
-    Assert.assertArrayEquals(partVals[0], parts.get(0).getValues().toArray(new String[2]));
-    Assert.assertArrayEquals(partVals[1], parts.get(1).getValues().toArray(new String[2]));
-
-    store.dropPartitions(DB, tableName, names);
-    List<Partition> afterDropParts = store.getPartitions(DB, tableName, -1);
-    Assert.assertEquals(0, afterDropParts.size());
-  }
-
-
-  @Test
-  public void dropPartition() throws Exception {
-    String tableName = "myparttable2";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> vals = Arrays.asList("fred");
-    StorageDescriptor psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/pc=fred");
-    Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
-        emptyParameters);
-    store.addPartition(part);
-
-    Assert.assertNotNull(store.getPartition(DB, tableName, vals));
-    store.dropPartition(DB, tableName, vals);
-    thrown.expect(NoSuchObjectException.class);
-    store.getPartition(DB, tableName, vals);
-  }
-
-  @Test
-  public void createIndex() throws Exception {
-    String tableName = "mytable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", ""));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    Map<String, String> params = new HashMap<String, String>();
-    params.put("key", "value");
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
-        serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    String indexName = "myindex";
-    String indexTableName = tableName + "__" + indexName + "__";
-    Index index = new Index(indexName, null, "default", tableName, startTime, startTime,
-        indexTableName, sd, emptyParameters, false);
-    store.addIndex(index);
-
-    Index ind = store.getIndex("default", tableName, indexName);
-    Assert.assertEquals(1, ind.getSd().getColsSize());
-    Assert.assertEquals("col1", ind.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", ind.getSd().getCols().get(0).getType());
-    Assert.assertEquals("", ind.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", ind.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", ind.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp", ind.getSd().getLocation());
-    Assert.assertEquals("input", ind.getSd().getInputFormat());
-    Assert.assertEquals("output", ind.getSd().getOutputFormat());
-    Assert.assertFalse(ind.getSd().isCompressed());
-    Assert.assertEquals(17, ind.getSd().getNumBuckets());
-    Assert.assertEquals(1, ind.getSd().getBucketColsSize());
-    Assert.assertEquals("bucketcol", ind.getSd().getBucketCols().get(0));
-    Assert.assertEquals(1, ind.getSd().getSortColsSize());
-    Assert.assertEquals("sortcol", ind.getSd().getSortCols().get(0).getCol());
-    Assert.assertEquals(1, ind.getSd().getSortCols().get(0).getOrder());
-    Assert.assertEquals(1, ind.getSd().getParametersSize());
-    Assert.assertEquals("value", ind.getSd().getParameters().get("key"));
-    Assert.assertEquals(indexName, ind.getIndexName());
-    Assert.assertNull(ind.getIndexHandlerClass());
-    Assert.assertEquals("default", ind.getDbName());
-    Assert.assertEquals(tableName, ind.getOrigTableName());
-    Assert.assertEquals(0, ind.getParametersSize());
-    Assert.assertEquals(startTime, ind.getCreateTime());
-    Assert.assertEquals(startTime, ind.getLastAccessTime());
-    Assert.assertEquals(false, ind.isDeferredRebuild());
-  }
-
-  @Test
-  public void alterIndex() throws Exception {
-    String tableName = "mytable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", ""));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    Map<String, String> params = new HashMap<String, String>();
-    params.put("key", "value");
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
-        serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    String indexName = "myindex";
-    Index index = new Index(indexName, null, "default", tableName, startTime, startTime,
-        tableName + "__" + indexName + "__", sd, emptyParameters, false);
-    store.addIndex(index);
-
-    startTime += 10;
-    index.setLastAccessTime(startTime);
-    store.alterIndex("default", tableName, indexName, index); 
-
-    Index ind = store.getIndex("default", tableName, indexName);
-    Assert.assertEquals(1, ind.getSd().getColsSize());
-    Assert.assertEquals("col1", ind.getSd().getCols().get(0).getName());
-    Assert.assertEquals("int", ind.getSd().getCols().get(0).getType());
-    Assert.assertEquals("", ind.getSd().getCols().get(0).getComment());
-    Assert.assertEquals("serde", ind.getSd().getSerdeInfo().getName());
-    Assert.assertEquals("seriallib", ind.getSd().getSerdeInfo().getSerializationLib());
-    Assert.assertEquals("file:/tmp", ind.getSd().getLocation());
-    Assert.assertEquals("input", ind.getSd().getInputFormat());
-    Assert.assertEquals("output", ind.getSd().getOutputFormat());
-    Assert.assertFalse(ind.getSd().isCompressed());
-    Assert.assertEquals(17, ind.getSd().getNumBuckets());
-    Assert.assertEquals(1, ind.getSd().getBucketColsSize());
-    Assert.assertEquals("bucketcol", ind.getSd().getBucketCols().get(0));
-    Assert.assertEquals(1, ind.getSd().getSortColsSize());
-    Assert.assertEquals("sortcol", ind.getSd().getSortCols().get(0).getCol());
-    Assert.assertEquals(1, ind.getSd().getSortCols().get(0).getOrder());
-    Assert.assertEquals(1, ind.getSd().getParametersSize());
-    Assert.assertEquals("value", ind.getSd().getParameters().get("key"));
-    Assert.assertEquals(indexName, ind.getIndexName());
-    Assert.assertNull(ind.getIndexHandlerClass());
-    Assert.assertEquals("default", ind.getDbName());
-    Assert.assertEquals(tableName, ind.getOrigTableName());
-    Assert.assertEquals(0, ind.getParametersSize());
-    Assert.assertEquals(startTime, ind.getLastAccessTime());
-    Assert.assertEquals(false, ind.isDeferredRebuild());
-  }
-
-  @Test
-  public void dropIndex() throws Exception {
-    String tableName = "mytable";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", ""));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    Map<String, String> params = new HashMap<String, String>();
-    params.put("key", "value");
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
-        serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
-    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    String indexName = "myindex";
-    Index index = new Index(indexName, null, "default", tableName, startTime, startTime,
-        tableName + "__" + indexName + "__", sd, emptyParameters, false);
-    store.addIndex(index);
-
-    store.dropIndex("default", tableName, indexName);
-
-  }
-
-  @Test
-  public void createRole() throws Exception {
-    int now = (int)System.currentTimeMillis()/1000;
-    String roleName = "myrole";
-    store.addRole(roleName, "me");
-
-    Role r = store.getRole(roleName);
-    Assert.assertEquals(roleName, r.getRoleName());
-    Assert.assertEquals("me", r.getOwnerName());
-    Assert.assertTrue(now <= r.getCreateTime());
-  }
-
-  @Test
-  public void dropRole() throws Exception {
-    String roleName = "anotherrole";
-    store.addRole(roleName, "me");
-
-    Role role = store.getRole(roleName);
-    Assert.assertNotNull(role);
-
-    store.removeRole(roleName);
-    thrown.expect(NoSuchObjectException.class);
-    store.getRole(roleName);
-  }
-
-  // Due to the way our mock stuff works, we can only insert one column at a time, so we'll test
-  // each stat type separately.  We'll test them together in the integration tests.
-  @Test
-  public void booleanTableStatistics() throws Exception {
-    // Add a boolean table stats for BOOLEAN_COL to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = booleanColStatsObjs.get(0);
-    BooleanColumnStatsData boolData = obj.getStatsData().getBooleanStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(BOOLEAN_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS, dataFromDB.getSetField());
-    // Compare BooleanColumnStatsData
-    BooleanColumnStatsData boolDataFromDB = dataFromDB.getBooleanStats();
-    Assert.assertEquals(boolData.getNumTrues(), boolDataFromDB.getNumTrues());
-    Assert.assertEquals(boolData.getNumFalses(), boolDataFromDB.getNumFalses());
-    Assert.assertEquals(boolData.getNumNulls(), boolDataFromDB.getNumNulls());
-  }
-
-  @Test
-  public void longTableStatistics() throws Exception {
-    createMockTable(LONG_TYPE);
-    // Add a long table stats for LONG_COL to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = longColStatsObjs.get(0);
-    LongColumnStatsData longData = obj.getStatsData().getLongStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(LONG_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, dataFromDB.getSetField());
-    // Compare LongColumnStatsData
-    LongColumnStatsData longDataFromDB = dataFromDB.getLongStats();
-    Assert.assertEquals(longData.getHighValue(), longDataFromDB.getHighValue());
-    Assert.assertEquals(longData.getLowValue(), longDataFromDB.getLowValue());
-    Assert.assertEquals(longData.getNumNulls(), longDataFromDB.getNumNulls());
-    Assert.assertEquals(longData.getNumDVs(), longDataFromDB.getNumDVs());
-  }
-
-  @Test
-  public void doubleTableStatistics() throws Exception {
-    createMockTable(DOUBLE_TYPE);
-    // Add a double table stats for DOUBLE_COL to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = doubleColStatsObjs.get(0);
-    DoubleColumnStatsData doubleData = obj.getStatsData().getDoubleStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(DOUBLE_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, dataFromDB.getSetField());
-    // Compare DoubleColumnStatsData
-    DoubleColumnStatsData doubleDataFromDB = dataFromDB.getDoubleStats();
-    Assert.assertEquals(doubleData.getHighValue(), doubleDataFromDB.getHighValue(), 0.01);
-    Assert.assertEquals(doubleData.getLowValue(), doubleDataFromDB.getLowValue(), 0.01);
-    Assert.assertEquals(doubleData.getNumNulls(), doubleDataFromDB.getNumNulls());
-    Assert.assertEquals(doubleData.getNumDVs(), doubleDataFromDB.getNumDVs());
-  }
-
-  @Test
-  public void stringTableStatistics() throws Exception {
-    createMockTable(STRING_TYPE);
-    // Add a string table stats for STRING_COL to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = stringColStatsObjs.get(0);
-    StringColumnStatsData stringData = obj.getStatsData().getStringStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(STRING_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, dataFromDB.getSetField());
-    // Compare StringColumnStatsData
-    StringColumnStatsData stringDataFromDB = dataFromDB.getStringStats();
-    Assert.assertEquals(stringData.getMaxColLen(), stringDataFromDB.getMaxColLen());
-    Assert.assertEquals(stringData.getAvgColLen(), stringDataFromDB.getAvgColLen(), 0.01);
-    Assert.assertEquals(stringData.getNumNulls(), stringDataFromDB.getNumNulls());
-    Assert.assertEquals(stringData.getNumDVs(), stringDataFromDB.getNumDVs());
-  }
-
-  @Test
-  public void binaryTableStatistics() throws Exception {
-    createMockTable(BINARY_TYPE);
-    // Add a binary table stats for BINARY_COL to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = binaryColStatsObjs.get(0);
-    BinaryColumnStatsData binaryData = obj.getStatsData().getBinaryStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(BINARY_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.BINARY_STATS, dataFromDB.getSetField());
-    // Compare BinaryColumnStatsData
-    BinaryColumnStatsData binaryDataFromDB = dataFromDB.getBinaryStats();
-    Assert.assertEquals(binaryData.getMaxColLen(), binaryDataFromDB.getMaxColLen());
-    Assert.assertEquals(binaryData.getAvgColLen(), binaryDataFromDB.getAvgColLen(), 0.01);
-    Assert.assertEquals(binaryData.getNumNulls(), binaryDataFromDB.getNumNulls());
-  }
-
-  @Test
-  public void decimalTableStatistics() throws Exception {
-    createMockTable(DECIMAL_TYPE);
-    // Add a decimal table stats for DECIMAL_COL to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for table level stats
-    ColumnStatisticsDesc desc = getMockTblColStatsDesc();
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = decimalColStatsObjs.get(0);
-    DecimalColumnStatsData decimalData = obj.getStatsData().getDecimalStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    store.updateTableColumnStatistics(stats);
-    // Get from DB
-    ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(DECIMAL_COL));
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
-    Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, dataFromDB.getSetField());
-    // Compare DecimalColumnStatsData
-    DecimalColumnStatsData decimalDataFromDB = dataFromDB.getDecimalStats();
-    Assert.assertEquals(decimalData.getHighValue(), decimalDataFromDB.getHighValue());
-    Assert.assertEquals(decimalData.getLowValue(), decimalDataFromDB.getLowValue());
-    Assert.assertEquals(decimalData.getNumNulls(), decimalDataFromDB.getNumNulls());
-    Assert.assertEquals(decimalData.getNumDVs(), decimalDataFromDB.getNumDVs());
-  }
-
-  @Test
-  public void booleanPartitionStatistics() throws Exception {
-    createMockTableAndPartition(BOOLEAN_TYPE, BOOLEAN_VAL);
-    // Add partition stats for: BOOLEAN_COL and partition: {PART_KEY, BOOLEAN_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, BOOLEAN_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = booleanColStatsObjs.get(0);
-    BooleanColumnStatsData boolData = obj.getStatsData().getBooleanStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(BOOLEAN_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS, dataFromDB.getSetField());
-    // Compare BooleanColumnStatsData
-    BooleanColumnStatsData boolDataFromDB = dataFromDB.getBooleanStats();
-    Assert.assertEquals(boolData.getNumTrues(), boolDataFromDB.getNumTrues());
-    Assert.assertEquals(boolData.getNumFalses(), boolDataFromDB.getNumFalses());
-    Assert.assertEquals(boolData.getNumNulls(), boolDataFromDB.getNumNulls());
-  }
-
-  @Test
-  public void longPartitionStatistics() throws Exception {
-    createMockTableAndPartition(INT_TYPE, INT_VAL);
-    // Add partition stats for: LONG_COL and partition: {PART_KEY, INT_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, INT_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = longColStatsObjs.get(0);
-    LongColumnStatsData longData = obj.getStatsData().getLongStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(INT_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, dataFromDB.getSetField());
-    // Compare LongColumnStatsData
-    LongColumnStatsData longDataFromDB = dataFromDB.getLongStats();
-    Assert.assertEquals(longData.getHighValue(), longDataFromDB.getHighValue());
-    Assert.assertEquals(longData.getLowValue(), longDataFromDB.getLowValue());
-    Assert.assertEquals(longData.getNumNulls(), longDataFromDB.getNumNulls());
-    Assert.assertEquals(longData.getNumDVs(), longDataFromDB.getNumDVs());
-  }
-
-  @Test
-  public void doublePartitionStatistics() throws Exception {
-    createMockTableAndPartition(DOUBLE_TYPE, DOUBLE_VAL);
-    // Add partition stats for: DOUBLE_COL and partition: {PART_KEY, DOUBLE_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, DOUBLE_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = doubleColStatsObjs.get(0);
-    DoubleColumnStatsData doubleData = obj.getStatsData().getDoubleStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(DOUBLE_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, dataFromDB.getSetField());
-    // Compare DoubleColumnStatsData
-    DoubleColumnStatsData doubleDataFromDB = dataFromDB.getDoubleStats();
-    Assert.assertEquals(doubleData.getHighValue(), doubleDataFromDB.getHighValue(), 0.01);
-    Assert.assertEquals(doubleData.getLowValue(), doubleDataFromDB.getLowValue(), 0.01);
-    Assert.assertEquals(doubleData.getNumNulls(), doubleDataFromDB.getNumNulls());
-    Assert.assertEquals(doubleData.getNumDVs(), doubleDataFromDB.getNumDVs());
-  }
-
-  @Test
-  public void stringPartitionStatistics() throws Exception {
-    createMockTableAndPartition(STRING_TYPE, STRING_VAL);
-    // Add partition stats for: STRING_COL and partition: {PART_KEY, STRING_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, STRING_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = stringColStatsObjs.get(0);
-    StringColumnStatsData stringData = obj.getStatsData().getStringStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(STRING_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, dataFromDB.getSetField());
-    // Compare StringColumnStatsData
-    StringColumnStatsData stringDataFromDB = dataFromDB.getStringStats();
-    Assert.assertEquals(stringData.getMaxColLen(), stringDataFromDB.getMaxColLen());
-    Assert.assertEquals(stringData.getAvgColLen(), stringDataFromDB.getAvgColLen(), 0.01);
-    Assert.assertEquals(stringData.getNumNulls(), stringDataFromDB.getNumNulls());
-    Assert.assertEquals(stringData.getNumDVs(), stringDataFromDB.getNumDVs());
-  }
-
-  @Test
-  public void binaryPartitionStatistics() throws Exception {
-    createMockTableAndPartition(BINARY_TYPE, BINARY_VAL);
-    // Add partition stats for: BINARY_COL and partition: {PART_KEY, BINARY_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, BINARY_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = binaryColStatsObjs.get(0);
-    BinaryColumnStatsData binaryData = obj.getStatsData().getBinaryStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(BINARY_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.BINARY_STATS, dataFromDB.getSetField());
-    // Compare BinaryColumnStatsData
-    BinaryColumnStatsData binaryDataFromDB = dataFromDB.getBinaryStats();
-    Assert.assertEquals(binaryData.getMaxColLen(), binaryDataFromDB.getMaxColLen());
-    Assert.assertEquals(binaryData.getAvgColLen(), binaryDataFromDB.getAvgColLen(), 0.01);
-    Assert.assertEquals(binaryData.getNumNulls(), binaryDataFromDB.getNumNulls());
-  }
-
-  @Test
-  public void decimalPartitionStatistics() throws Exception {
-    createMockTableAndPartition(DECIMAL_TYPE, DECIMAL_VAL);
-    // Add partition stats for: DECIMAL_COL and partition: {PART_KEY, DECIMAL_VAL} to DB
-    // Because of the way our mock implementation works we actually need to not create the table
-    // before we set statistics on it.
-    ColumnStatistics stats = new ColumnStatistics();
-    // Get a default ColumnStatisticsDesc for partition level stats
-    ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, DECIMAL_VAL);
-    stats.setStatsDesc(desc);
-    // Get one of the pre-created ColumnStatisticsObj
-    ColumnStatisticsObj obj = decimalColStatsObjs.get(0);
-    DecimalColumnStatsData decimalData = obj.getStatsData().getDecimalStats();
-    // Add to DB
-    stats.addToStatsObj(obj);
-    List<String> parVals = new ArrayList<String>();
-    parVals.add(DECIMAL_VAL);
-    store.updatePartitionColumnStatistics(stats, parVals);
-    // Get from DB
-    List<String> partNames = new ArrayList<String>();
-    partNames.add(desc.getPartName());
-    List<String> colNames = new ArrayList<String>();
-    colNames.add(obj.getColName());
-    List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
-    // Compare ColumnStatisticsDesc
-    Assert.assertEquals(1, statsFromDB.size());
-    Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
-    Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
-    Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
-    Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
-    // Compare ColumnStatisticsObj
-    Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
-    ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
-    ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
-    // Compare ColumnStatisticsData
-    Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, dataFromDB.getSetField());
-    // Compare DecimalColumnStatsData
-    DecimalColumnStatsData decimalDataFromDB = dataFromDB.getDecimalStats();
-    Assert.assertEquals(decimalData.getHighValue(), decimalDataFromDB.getHighValue());
-    Assert.assertEquals(decimalData.getLowValue(), decimalDataFromDB.getLowValue());
-    Assert.assertEquals(decimalData.getNumNulls(), decimalDataFromDB.getNumNulls());
-    Assert.assertEquals(decimalData.getNumDVs(), decimalDataFromDB.getNumDVs());
-  }
-
-  @Test
-  public void createTableWithPrimaryKey() throws Exception {
-    String tableName = "pktable";
-    String pkName = "test_pk";
-    String pkColNames[] = { "col0" };
-    Table table = createMultiColumnTable(tableName, "int");
-
-    List<SQLPrimaryKey> pk = Arrays.asList(
-        new SQLPrimaryKey(DB, tableName, pkColNames[0], 0, pkName, true, false, true));
-
-    store.createTableWithConstraints(table, pk, null, null, null);
-
-    pk = store.getPrimaryKeys(DB, tableName);
-
-    Assert.assertNotNull(pk);
-    Assert.assertEquals(1, pk.size());
-    Assert.assertEquals(DB, pk.get(0).getTable_db());
-    Assert.assertEquals(tableName, pk.get(0).getTable_name());
-    Assert.assertEquals(pkColNames[0], pk.get(0).getColumn_name());
-    Assert.assertEquals(0, pk.get(0).getKey_seq());
-    Assert.assertEquals(pkName, pk.get(0).getPk_name());
-    Assert.assertTrue(pk.get(0).isEnable_cstr());
-    Assert.assertFalse(pk.get(0).isValidate_cstr());
-    Assert.assertTrue(pk.get(0).isRely_cstr());
-
-    // Drop the primary key
-    store.dropConstraint(DB, tableName, pkName);
-
-    pk = store.getPrimaryKeys(DB, tableName);
-    Assert.assertNull(pk);
-  }
-
-  @Test
-  public void createTableWithForeignKey() throws Exception {
-    String tableName = "fktable";
-    String pkTable = "pktable";
-    String pkName = "test_pk";
-    String fkName = "test_fk";
-    String fkColNames[] = { "col0" };
-    String pkColNames[] = { "pcol0" };
-    Table table = createMultiColumnTable(tableName, "int");
-
-    List<SQLForeignKey> fk = Arrays.asList(
-        new SQLForeignKey(DB, pkTable, pkColNames[0], DB, tableName, fkColNames[0], 0, 1, 2,
-            fkName, pkName, true, false, false));
-
-    store.createTableWithConstraints(table, null, fk, null, null);
-
-    fk = store.getForeignKeys(DB, pkTable, DB, tableName);
-
-    Assert.assertNotNull(fk);
-    Assert.assertEquals(1, fk.size());
-    Assert.assertEquals(DB, fk.get(0).getPktable_db());
-    Assert.assertEquals(pkTable, fk.get(0).getPktable_name());
-    Assert.assertEquals(pkColNames[0], fk.get(0).getPkcolumn_name());
-    Assert.assertEquals(DB, fk.get(0).getFktable_db());
-    Assert.assertEquals(tableName, fk.get(0).getFktable_name());
-    Assert.assertEquals(fkColNames[0], fk.get(0).getFkcolumn_name());
-    Assert.assertEquals(0, fk.get(0).getKey_seq());
-    Assert.assertEquals(1, fk.get(0).getUpdate_rule());
-    Assert.assertEquals(2, fk.get(0).getDelete_rule());
-    Assert.assertEquals(fkName, fk.get(0).getFk_name());
-    Assert.assertEquals(pkName, fk.get(0).getPk_name());
-    Assert.assertTrue(fk.get(0).isEnable_cstr());
-    Assert.assertFalse(fk.get(0).isValidate_cstr());
-    Assert.assertFalse(fk.get(0).isRely_cstr());
-  }
-
-  // Test that we can add a primary key with multiple columns
-  @Test
-  public void addMultiColPrimaryKey() throws Exception {
-    String tableName = "mcpktable";
-    String pkName = "test_pk";
-    String pkColNames[] = { "col0", "col1", "col2" };
-    Table table = createMultiColumnTable(tableName, "int", "varchar(32)", "decimal(10,2)");
-
-    List<SQLPrimaryKey> pk = Arrays.asList(
-        new SQLPrimaryKey(DB, tableName, pkColNames[1], 0, pkName, false, true, true),
-        new SQLPrimaryKey(DB, tableName, pkColNames[2], 1, pkName, false, true, true)
-    );
-
-    store.createTable(table);
-    store.addPrimaryKeys(pk);
-
-    Assert.assertNotNull(pk);
-    Assert.assertEquals(2, pk.size());
-    SQLPrimaryKey[] sorted = pk.toArray(new SQLPrimaryKey[2]);
-    Arrays.sort(sorted, new Comparator<SQLPrimaryKey>() {
-      @Override
-      public int compare(SQLPrimaryKey o1, SQLPrimaryKey o2) {
-        return o1.getColumn_name().compareTo(o2.getColumn_name());
-      }
-    });
-    for (int i = 0; i < 2; i++) {
-      Assert.assertEquals(DB, sorted[i].getTable_db());
-      Assert.assertEquals(tableName, sorted[i].getTable_name());
-      Assert.assertEquals(pkColNames[i+1], sorted[i].getColumn_name());
-      Assert.assertEquals(i, sorted[i].getKey_seq());
-      Assert.assertEquals(pkName, sorted[i].getPk_name());
-      Assert.assertFalse(sorted[i].isEnable_cstr());
-      Assert.assertTrue(sorted[i].isValidate_cstr());
-      Assert.assertTrue(sorted[i].isRely_cstr());
-    }
-
-  }
-
-  // Test that we can create a foreign key with multiple columns
-  @Test
-  public void addMultiColForeignKey() throws Exception {
-    String tableName = "mcfktable";
-    String pkTable = "pktable";
-    String pkName = "test_pk";
-    String fkName = "test_fk";
-    String fkColNames[] = { "col0", "col1", "col2" };
-    String pkColNames[] = { "pcol0", "pcol1" };
-    Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
-    List<SQLForeignKey> fk = Arrays.asList(
-        new SQLForeignKey(DB, pkTable, pkColNames[0], DB, tableName, fkColNames[1], 0, 1, 2,
-            fkName, pkName, true, false, false),
-        new SQLForeignKey(DB, pkTable, pkColNames[1], DB, tableName, fkColNames[2], 1, 1, 2,
-            fkName, pkName, true, false, false)
-        );
-
-    store.createTable(table);
-    store.addForeignKeys(fk);
-
-    fk = store.getForeignKeys(DB, pkTable, DB, tableName);
-
-    Assert.assertNotNull(fk);
-    Assert.assertEquals(2, fk.size());
-    SQLForeignKey[] sorted = fk.toArray(new SQLForeignKey[2]);
-    Arrays.sort(sorted, new Comparator<SQLForeignKey>() {
-      @Override
-      public int compare(SQLForeignKey o1, SQLForeignKey o2) {
-        if (o1.getFk_name().equals(o2.getFk_name())) {
-          return o1.getFkcolumn_name().compareTo(o2.getFkcolumn_name());
-        } else {
-          return o1.getFk_name().compareTo(o2.getFk_name());
-        }
-      }
-    });
-
-    for (int i = 0; i < 2; i++) {
-      Assert.assertEquals(DB, sorted[i].getPktable_db());
-      Assert.assertEquals(pkTable, sorted[i].getPktable_name());
-      Assert.assertEquals(pkColNames[i], sorted[i].getPkcolumn_name());
-      Assert.assertEquals(DB, sorted[i].getFktable_db());
-      Assert.assertEquals(tableName, sorted[i].getFktable_name());
-      Assert.assertEquals(fkColNames[i+1], sorted[i].getFkcolumn_name());
-      Assert.assertEquals(i, sorted[i].getKey_seq());
-      Assert.assertEquals(1, sorted[i].getUpdate_rule());
-      Assert.assertEquals(2, sorted[i].getDelete_rule());
-      Assert.assertEquals(fkName, sorted[i].getFk_name());
-      Assert.assertEquals(pkName, sorted[i].getPk_name());
-      Assert.assertTrue(sorted[i].isEnable_cstr());
-      Assert.assertFalse(sorted[i].isValidate_cstr());
-      Assert.assertFalse(sorted[i].isRely_cstr());
-    }
-
-  }
-
-  // Test that we can add 2 foreign keys at once
-  @Test
-  public void addMultiForeignKeys() throws Exception {
-    String tableName = "mcfktable";
-    String pkTable = "pktable";
-    String pkTable2 = "pktable2";
-    String pkName = "test_pk";
-    String pkName2 = "test_pk2";
-    String fkName = "test_fk";
-    String fkName2 = "test_fk2";
-    String fkColNames[] = { "col0", "col1", "col2" };
-    String pkColNames[] = { "pcol0", "pcol1" };
-    String pkColNames2[] = { "p2col0" };
-    Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
-    List<SQLForeignKey> fk = Arrays.asList(
-        new SQLForeignKey(DB, pkTable, pkColNames[0], DB, tableName, fkColNames[1], 0, 1, 2,
-            fkName, pkName, true, false, true),
-        new SQLForeignKey(DB, pkTable, pkColNames[1], DB, tableName, fkColNames[2], 1, 1, 2,
-            fkName, pkName, true, false, true),
-        new SQLForeignKey(DB, pkTable2, pkColNames2[0], DB, tableName, fkColNames[0], 0, 1, 2,
-            fkName2, pkName2, true, false, true)
-    );
-
-    store.createTable(table);
-    store.addForeignKeys(fk);
-
-    fk = store.getForeignKeys(DB, pkTable, DB, tableName);
-
-    Assert.assertNotNull(fk);
-    Assert.assertEquals(2, fk.size());
-    SQLForeignKey[] sorted = fk.toArray(new SQLForeignKey[2]);
-    Arrays.sort(sorted, new Comparator<SQLForeignKey>() {
-      @Override
-      public int compare(SQLForeignKey o1, SQLForeignKey o2) {
-        if (o1.getFk_name().equals(o2.getFk_name())) {
-          return o1.getFkcolumn_name().compareTo(o2.getFkcolumn_name());
-        } else {
-          return o1.getFk_name().compareTo(o2.getFk_name());
-        }
-      }
-    });
-
-    for (int i = 0; i < 2; i++) {
-      Assert.assertEquals(DB, sorted[i].getPktable_db());
-      Assert.assertEquals(pkTable, sorted[i].getPktable_name());
-      Assert.assertEquals(pkColNames[i], sorted[i].getPkcolumn_name());
-      Assert.assertEquals(DB, sorted[i].getFktable_db());
-      Assert.assertEquals(tableName, sorted[i].getFktable_name());
-      Assert.assertEquals(fkColNames[i+1], sorted[i].getFkcolumn_name());
-      Assert.assertEquals(i, sorted[i].getKey_seq());
-      Assert.assertEquals(1, sorted[i].getUpdate_rule());
-      Assert.assertEquals(2, sorted[i].getDelete_rule());
-      Assert.assertEquals(fkName, sorted[i].getFk_name());
-      Assert.assertEquals(pkName, sorted[i].getPk_name());
-      Assert.assertTrue(sorted[i].isEnable_cstr());
-      Assert.assertFalse(sorted[i].isValidate_cstr());
-      Assert.assertTrue(sorted[i].isRely_cstr());
-    }
-    fk = store.getForeignKeys(DB, pkTable2, DB, tableName);
-    Assert.assertNotNull(fk);
-    Assert.assertEquals(1, fk.size());
-    Assert.assertEquals(DB, fk.get(0).getPktable_db());
-    Assert.assertEquals(pkTable2, fk.get(0).getPktable_name());
-    Assert.assertEquals(pkColNames2[0], fk.get(0).getPkcolumn_name());
-    Assert.assertEquals(DB, fk.get(0).getFktable_db());
-    Assert.assertEquals(tableName, fk.get(0).getFktable_name());
-    Assert.assertEquals(fkColNames[0], fk.get(0).getFkcolumn_name());
-    Assert.assertEquals(0, fk.get(0).getKey_seq());
-    Assert.assertEquals(1, fk.get(0).getUpdate_rule());
-    Assert.assertEquals(2, fk.get(0).getDelete_rule());
-    Assert.assertEquals(fkName2, fk.get(0).getFk_name());
-    Assert.assertEquals(pkName2, fk.get(0).getPk_name());
-    Assert.assertTrue(fk.get(0).isEnable_cstr());
-    Assert.assertFalse(fk.get(0).isValidate_cstr());
-    Assert.assertTrue(fk.get(0).isRely_cstr());
-
-  }
-
-  // Test that we can add a foreign key when one already exists
-  @Test
-  public void addSecondForeignKeys() throws Exception {
-    String tableName = "mcfktable";
-    String pkTable = "pktable";
-    String pkTable2 = "pktable2";
-    String pkName = "test_pk";
-    String pkName2 = "test_pk2";
-    String fkName = "test_fk";
-    String fkName2 = "test_fk2";
-    String fkColNames[] = { "col0", "col1", "col2" };
-    String pkColNames[] = { "pcol0", "pcol1" };
-    String pkColNames2[] = { "p2col0" };
-    Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
-    List<SQLForeignKey> fk = Arrays.asList(
-        new SQLForeignKey(DB, pkTable, pkColNames[0], DB, tableName, fkColNames[1], 0, 1, 2,
-            fkName, pkName, true, false, true),
-        new SQLForeignKey(DB, pkTable, pkColNames[1], DB, tableName, fkColNames[2], 1, 1, 2,
-            fkName, pkName, true, false, true)
-    );
-
-    store.createTable(table);
-    store.addForeignKeys(fk);
-
-    fk = Arrays.asList(
-        new SQLForeignKey(DB, pkTable2, pkColNames2[0], DB, tableName, fkColNames[0], 0, 1, 2,
-            fkName2, pkName2, true, false, true)
-    );
-    store.addForeignKeys(fk);
-
-    fk = store.getForeignKeys(DB, pkTable, DB, tableName);
-
-    Assert.assertNotNull(fk);
-    Assert.assertEquals(2, fk.size());
-    SQLForeignKey[] sorted = fk.toArray(new SQLForeignKey[2]);
-    Arrays.sort(sorted, new Comparator<SQLForeignKey>() {
-      @Override
-      public int compare(SQLForeignKey o1, SQLForeignKey o2) {
-        if (o1.getFk_name().equals(o2.getFk_name())) {
-          return o1.getFkcolumn_name().compareTo(o2.getFkcolumn_name());
-        } else {
-          return o1.getFk_name().compareTo(o2.getFk_name());
-        }
-      }
-    });
-
-    for (int i = 0; i < 2; i++) {
-      Assert.assertEquals(DB, sorted[i].getPktable_db());
-      Assert.assertEquals(pkTable, sorted[i].getPktable_name());
-      Assert.assertEquals(pkColNames[i], sorted[i].getPkcolumn_name());
-      Assert.assertEquals(DB, sorted[i].getFktable_db());
-      Assert.assertEquals(tableName, sorted[i].getFktable_name());
-      Assert.assertEquals(fkColNames[i+1], sorted[i].getFkcolumn_name());
-      Assert.assertEquals(i, sorted[i].getKey_seq());
-      Assert.assertEquals(1, sorted[i].getUpdate_rule());
-      Assert.assertEquals(2, sorted[i].getDelete_rule());
-      Assert.assertEquals(fkName, sorted[i].getFk_name());
-      Assert.assertEquals(pkName, sorted[i].getPk_name());
-      Assert.assertTrue(sorted[i].isEnable_cstr());
-      Assert.assertFalse(sorted[i].isValidate_cstr());
-      Assert.assertTrue(sorted[i].isRely_cstr());
-    }
-
-    fk = store.getForeignKeys(DB, pkTable2, DB, tableName);
-    Assert.assertNotNull(fk);
-    Assert.assertEquals(1, fk.size());
-    Assert.assertEquals(DB, fk.get(0).getPktable_db());
-    Assert.assertEquals(pkTable2, fk.get(0).getPktable_name());
-    Assert.assertEquals(pkColNames2[0], fk.get(0).getPkcolumn_name());
-    Assert.assertEquals(DB, fk.get(0).getFktable_db());
-    Assert.assertEquals(tableName, fk.get(0).getFktable_name());
-    Assert.assertEquals(fkColNames[0], fk.get(0).getFkcolumn_name());
-    Assert.assertEquals(0, fk.get(0).getKey_seq());
-    Assert.assertEquals(1, fk.get(0).getUpdate_rule());
-    Assert.assertEquals(2, fk.get(0).getDelete_rule());
-    Assert.assertEquals(fkName2, fk.get(0).getFk_name());
-    Assert.assertEquals(pkName2, fk.get(0).getPk_name());
-    Assert.assertTrue(fk.get(0).isEnable_cstr());
-    Assert.assertFalse(fk.get(0).isValidate_cstr());
-    Assert.assertTrue(fk.get(0).isRely_cstr());
-
-    // Check that passing null gets all the foreign keys
-    fk = store.getForeignKeys(null, null, DB, tableName);
-    Assert.assertNotNull(fk);
-    Assert.assertEquals(3, fk.size());
-
-    store.dropConstraint(DB, tableName, fkName);
-
-    fk = store.getForeignKeys(DB, pkTable2, DB, tableName);
-    Assert.assertNotNull(fk);
-    Assert.assertEquals(1, fk.size());
-    Assert.assertEquals(DB, fk.get(0).getPktable_db());
-    Assert.assertEquals(pkTable2, fk.get(0).getPktable_name());
-    Assert.assertEquals(pkColNames2[0], fk.get(0).getPkcolumn_name());
-    Assert.assertEquals(DB, fk.get(0).getFktable_db());
-    Assert.assertEquals(tableName, fk.get(0).getFktable_name());
-    Assert.assertEquals(fkColNames[0], fk.get(0).getFkcolumn_name());
-    Assert.assertEquals(0, fk.get(0).getKey_seq());
-    Assert.assertEquals(1, fk.get(0).getUpdate_rule());
-    Assert.assertEquals(2, fk.get(0).getDelete_rule());
-    Assert.assertEquals(fkName2, fk.get(0).getFk_name());
-    Assert.assertEquals(pkName2, fk.get(0).getPk_name());
-    Assert.assertTrue(fk.get(0).isEnable_cstr());
-    Assert.assertFalse(fk.get(0).isValidate_cstr());
-    Assert.assertTrue(fk.get(0).isRely_cstr());
-
-    store.dropConstraint(DB, tableName, fkName2);
-
-    fk = store.getForeignKeys(DB, pkTable2, DB, tableName);
-    Assert.assertNull(fk);
-  }
-
-  // Try adding a primary key when one already exists
-  @Test(expected= MetaException.class)
-  public void doublePrimaryKey() throws Exception {
-    String tableName = "pktable";
-    String pkName = "test_pk";
-    String pkColNames[] = { "col0" };
-    Table table = createMultiColumnTable(tableName, "int");
-
-    List<SQLPrimaryKey> pk = Arrays.asList(
-        new SQLPrimaryKey(DB, tableName, pkColNames[0], 0, pkName, true, false, true));
-
-    store.createTableWithConstraints(table, pk, null, null, null);
-
-    store.addPrimaryKeys(pk);
-  }
-
-  @Test
-  public void createTableWithUniqueConstraint() throws Exception {
-    String tableName = "uktable";
-    String ukName = "test_uk";
-    String ukColNames[] = { "col0" };
-    Table table = createMultiColumnTable(tableName, "int");
-
-    List<SQLUniqueConstraint> uk = Arrays.asList(
-        new SQLUniqueConstraint(DB, tableName, ukColNames[0], 0, ukName, true, false, true));
-
-    store.createTableWithConstraints(table, null, null, uk, null);
-
-    uk = store.getUniqueConstraints(DB, tableName);
-
-    Assert.assertNotNull(uk);
-    Assert.assertEquals(1, uk.size());
-    Assert.assertEquals(DB, uk.get(0).getTable_db());
-    Assert.assertEquals(tableName, uk.get(0).getTable_name());
-    Assert.assertEquals(ukColNames[0], uk.get(0).getColumn_name());
-    Assert.assertEquals(0, uk.get(0).getKey_seq());
-    Assert.assertEquals(ukName, uk.get(0).getUk_name());
-    Assert.assertTrue(uk.get(0).isEnable_cstr());
-    Assert.assertFalse(uk.get(0).isValidate_cstr());
-    Assert.assertTrue(uk.get(0).isRely_cstr());
-
-    // Drop the unique constraint
-    store.dropConstraint(DB, tableName, ukName);
-
-    uk = store.getUniqueConstraints(DB, tableName);
-    Assert.assertNull(uk);
-  }
-
-  @Test
-  public void addMultiUniqueConstraints() throws Exception {
-    String tableName = "mcuktable";
-    String ukName = "test_uk";
-    String ukName2 = "test_uk2";
-    String ukColNames[] = { "col0", "col1" };
-    Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
-    List<SQLUniqueConstraint> uks = Arrays.asList(
-        new SQLUniqueConstraint(DB, tableName, ukColNames[0], 0, ukName, true, false, true),
-        new SQLUniqueConstraint(DB, tableName, ukColNames[1], 0, ukName2, true, false, true)
-    );
-
-    store.createTable(table);
-    store.addUniqueConstraints(uks);
-
-    uks = store.getUniqueConstraints(DB, tableName);
-
-    Assert.assertNotNull(uks);
-    Assert.assertEquals(2, uks.size());
-    SQLUniqueConstraint[] sorted = uks.toArray(new SQLUniqueConstraint[2]);
-    Arrays.sort(sorted, new Comparator<SQLUniqueConstraint>() {
-      @Override
-      public int compare(SQLUniqueConstraint o1, SQLUniqueConstraint o2) {
-        if (o1.getUk_name().equals(o2.getUk_name())) {
-          return o1.getColumn_name().compareTo(o2.getColumn_name());
-        } else {
-          return o1.getUk_name().compareTo(o2.getUk_name());
-        }
-      }
-    });
-
-    Assert.assertEquals(DB, sorted[0].getTable_db());
-    Assert.assertEquals(tableName, sorted[0].getTable_name());
-    Assert.assertEquals(ukColNames[0], sorted[0].getColumn_name());
-    Assert.assertEquals(0, sorted[0].getKey_seq());
-    Assert.assertEquals(ukName, sorted[0].getUk_name());
-    Assert.assertTrue(sorted[0].isEnable_cstr());
-    Assert.assertFalse(sorted[0].isValidate_cstr());
-    Assert.assertTrue(sorted[0].isRely_cstr());
-
-    Assert.assertEquals(DB, sorted[1].getTable_db());
-    Assert.assertEquals(tableName, sorted[1].getTable_name());
-    Assert.assertEquals(ukColNames[1], sorted[1].getColumn_name());
-    Assert.assertEquals(0, sorted[1].getKey_seq());
-    Assert.assertEquals(ukName2, sorted[1].getUk_name());
-    Assert.assertTrue(sorted[1].isEnable_cstr());
-    Assert.assertFalse(sorted[1].isValidate_cstr());
-    Assert.assertTrue(sorted[1].isRely_cstr());
-  }
-
-  @Test
-  public void addMultiNotNullConstraints() throws Exception {
-    String tableName = "mcnntable";
-    String nnName = "test_nn";
-    String nnName2 = "test_nn2";
-    String nnColNames[] = { "col0", "col1" };
-    Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
-    List<SQLNotNullConstraint> nns = Arrays.asList(
-        new SQLNotNullConstraint(DB, tableName, nnColNames[0], nnName, true, false, t

<TRUNCATED>

[10/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
deleted file mode 100644
index ab6457e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ /dev/null
@@ -1,2942 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterators;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.RegexStringComparator;
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hive.common.ObjectPair;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.hbase.PartitionKeyComparator.Operator;
-import org.apache.hive.common.util.BloomFilter;
-import org.apache.thrift.TBase;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.protocol.TSimpleJSONProtocol;
-import org.apache.thrift.transport.TMemoryBuffer;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-
-
-/**
- * Class to manage storing object in and reading them from HBase.
- */
-public class HBaseReadWrite implements MetadataStore {
-
-  final static String AGGR_STATS_TABLE = "HBMS_AGGR_STATS";
-  final static String DB_TABLE = "HBMS_DBS";
-  final static String FUNC_TABLE = "HBMS_FUNCS";
-  final static String GLOBAL_PRIVS_TABLE = "HBMS_GLOBAL_PRIVS";
-  final static String PART_TABLE = "HBMS_PARTITIONS";
-  final static String ROLE_TABLE = "HBMS_ROLES";
-  final static String SD_TABLE = "HBMS_SDS";
-  final static String SECURITY_TABLE = "HBMS_SECURITY";
-  final static String SEQUENCES_TABLE = "HBMS_SEQUENCES";
-  final static String TABLE_TABLE = "HBMS_TBLS";
-  final static String INDEX_TABLE = "HBMS_INDEX";
-  final static String USER_TO_ROLE_TABLE = "HBMS_USER_TO_ROLE";
-  final static String FILE_METADATA_TABLE = "HBMS_FILE_METADATA";
-  final static byte[] CATALOG_CF = "c".getBytes(HBaseUtils.ENCODING);
-  final static byte[] STATS_CF = "s".getBytes(HBaseUtils.ENCODING);
-  final static String NO_CACHE_CONF = "no.use.cache";
-  /**
-   * List of tables in HBase
-   */
-  public final static String[] tableNames = { AGGR_STATS_TABLE, DB_TABLE, FUNC_TABLE,
-                                              GLOBAL_PRIVS_TABLE, PART_TABLE, USER_TO_ROLE_TABLE,
-                                              ROLE_TABLE, SD_TABLE, SECURITY_TABLE, SEQUENCES_TABLE,
-                                              TABLE_TABLE, INDEX_TABLE, FILE_METADATA_TABLE };
-  public final static Map<String, List<byte[]>> columnFamilies = new HashMap<> (tableNames.length);
-
-  static {
-    columnFamilies.put(AGGR_STATS_TABLE, Arrays.asList(CATALOG_CF));
-    columnFamilies.put(DB_TABLE, Arrays.asList(CATALOG_CF));
-    columnFamilies.put(FUNC_TABLE, Arrays.asList(CATALOG_CF));
-    columnFamilies.put(GLOBAL_PRIVS_TABLE, Arrays.asList(CATALOG_CF));
-    columnFamilies.put(PART_TABLE, Arrays.asList(CATALOG_CF, STATS_CF));
-    columnFamilies.put(USER_TO_ROLE_TABLE, Arrays.asList(CATALOG_CF));
-    columnFamilies.put(ROLE_TABLE, Arrays.asList(CATALOG_CF));
-    columnFamilies.put(SD_TABLE, Arrays.asList(CATALOG_CF));
-    columnFamilies.put(SECURITY_TABLE, Arrays.asList(CATALOG_CF));
-    columnFamilies.put(SEQUENCES_TABLE, Arrays.asList(CATALOG_CF));
-    columnFamilies.put(TABLE_TABLE, Arrays.asList(CATALOG_CF, STATS_CF));
-    columnFamilies.put(INDEX_TABLE, Arrays.asList(CATALOG_CF, STATS_CF));
-    // Stats CF will contain PPD stats.
-    columnFamilies.put(FILE_METADATA_TABLE, Arrays.asList(CATALOG_CF, STATS_CF));
-  }
-
-  final static byte[] MASTER_KEY_SEQUENCE = "master_key".getBytes(HBaseUtils.ENCODING);
-  // The change version functionality uses the sequences table, but we don't want to give the
-  // caller complete control over the sequence name as they might inadvertently clash with one of
-  // our sequence keys, so add a prefix to their topic name.
-
-  final static byte[] AGGR_STATS_BLOOM_COL = "b".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] AGGR_STATS_STATS_COL = "s".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] CATALOG_COL = "c".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] ROLES_COL = "roles".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] REF_COUNT_COL = "ref".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] DELEGATION_TOKEN_COL = "dt".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] MASTER_KEY_COL = "mk".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] PRIMARY_KEY_COL = "pk".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] FOREIGN_KEY_COL = "fk".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] UNIQUE_CONSTRAINT_COL = "uk".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] NOT_NULL_CONSTRAINT_COL = "nn".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] GLOBAL_PRIVS_KEY = "gp".getBytes(HBaseUtils.ENCODING);
-  private final static byte[] SEQUENCES_KEY = "seq".getBytes(HBaseUtils.ENCODING);
-  private final static int TABLES_TO_CACHE = 10;
-  // False positives are very bad here because they cause us to invalidate entries we shouldn't.
-  // Space used and # of hash functions grows in proportion to ln of num bits so a 10x increase
-  // in accuracy doubles the required space and number of hash functions.
-  private final static double STATS_BF_ERROR_RATE = 0.001;
-
-  @VisibleForTesting final static String TEST_CONN = "test_connection";
-  private static HBaseConnection testConn;
-
-  static final private Logger LOG = LoggerFactory.getLogger(HBaseReadWrite.class.getName());
-
-  private static ThreadLocal<HBaseReadWrite> self = new ThreadLocal<HBaseReadWrite>() {
-    @Override
-    protected HBaseReadWrite initialValue() {
-      if (staticConf == null) {
-        throw new RuntimeException("Attempt to create HBaseReadWrite with no configuration set");
-      }
-      return new HBaseReadWrite(staticConf);
-    }
-  };
-
-  private static boolean tablesCreated = false;
-  private static Configuration staticConf = null;
-
-  private final Configuration conf;
-  private HBaseConnection conn;
-  private MessageDigest md;
-  private ObjectCache<ObjectPair<String, String>, Table> tableCache;
-  private ObjectCache<ByteArrayWrapper, StorageDescriptor> sdCache;
-  private PartitionCache partCache;
-  private StatsCache statsCache;
-  private Counter tableHits;
-  private Counter tableMisses;
-  private Counter tableOverflows;
-  private Counter partHits;
-  private Counter partMisses;
-  private Counter partOverflows;
-  private Counter sdHits;
-  private Counter sdMisses;
-  private Counter sdOverflows;
-  private List<Counter> counters;
-  // roleCache doesn't use ObjectCache because I don't want to limit the size.  I am assuming
-  // that the number of roles will always be small (< 100) so caching the whole thing should not
-  // be painful.
-  private final Map<String, HbaseMetastoreProto.RoleGrantInfoList> roleCache;
-  boolean entireRoleTableInCache;
-
-  /**
-   * Set the configuration for all HBaseReadWrite instances.
-   * @param configuration Configuration object
-   */
-  public static synchronized void setConf(Configuration configuration) {
-    if (staticConf == null) {
-      staticConf = configuration;
-    } else {
-      LOG.info("Attempt to set conf when it has already been set.");
-    }
-  }
-
-  /**
-   * Get the instance of HBaseReadWrite for the current thread.  This can only be called after
-   * {@link #setConf} has been called. Woe betide you if that's not the case.
-   * @return thread's instance of HBaseReadWrite
-   */
-  static HBaseReadWrite getInstance() {
-    if (staticConf == null) {
-      throw new RuntimeException("Must set conf object before getting an instance");
-    }
-    return self.get();
-  }
-
-  public Configuration getConf() {
-    return conf;
-  }
-
-  private HBaseReadWrite(Configuration configuration) {
-    conf = configuration;
-    HBaseConfiguration.addHbaseResources(conf);
-
-    try {
-      String connClass = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS);
-      if (TEST_CONN.equals(connClass)) {
-        conn = testConn;
-        LOG.debug("Using test connection.");
-      } else {
-        LOG.debug("Instantiating connection class " + connClass);
-        Class c = Class.forName(connClass);
-        Object o = c.newInstance();
-        if (HBaseConnection.class.isAssignableFrom(o.getClass())) {
-          conn = (HBaseConnection) o;
-        } else {
-          throw new IOException(connClass + " is not an instance of HBaseConnection.");
-        }
-        conn.setConf(conf);
-        conn.connect();
-      }
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-
-    try {
-      md = MessageDigest.getInstance("MD5");
-    } catch (NoSuchAlgorithmException e) {
-      throw new RuntimeException(e);
-    }
-    int totalCatalogObjectsToCache =
-        HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTORE_HBASE_CATALOG_CACHE_SIZE);
-
-    tableHits = new Counter("table cache hits");
-    tableMisses = new Counter("table cache misses");
-    tableOverflows = new Counter("table cache overflows");
-    partHits = new Counter("partition cache hits");
-    partMisses = new Counter("partition cache misses");
-    partOverflows = new Counter("partition cache overflows");
-    sdHits = new Counter("storage descriptor cache hits");
-    sdMisses = new Counter("storage descriptor cache misses");
-    sdOverflows = new Counter("storage descriptor cache overflows");
-    counters = new ArrayList<>();
-    counters.add(tableHits);
-    counters.add(tableMisses);
-    counters.add(tableOverflows);
-    counters.add(partHits);
-    counters.add(partMisses);
-    counters.add(partOverflows);
-    counters.add(sdHits);
-    counters.add(sdMisses);
-    counters.add(sdOverflows);
-
-    // Give 1% of catalog cache space to storage descriptors
-    // (storage descriptors are shared, so 99% should be the same for a given table)
-    int sdsCacheSize = totalCatalogObjectsToCache / 100;
-    if (conf.getBoolean(NO_CACHE_CONF, false)) {
-      tableCache = new BogusObjectCache<>();
-      sdCache = new BogusObjectCache<>();
-      partCache = new BogusPartitionCache();
-    } else {
-      tableCache = new ObjectCache<>(TABLES_TO_CACHE, tableHits, tableMisses, tableOverflows);
-      sdCache = new ObjectCache<>(sdsCacheSize, sdHits, sdMisses, sdOverflows);
-      partCache = new PartitionCache(totalCatalogObjectsToCache, partHits, partMisses, partOverflows);
-    }
-    statsCache = StatsCache.getInstance(conf);
-    roleCache = new HashMap<>();
-    entireRoleTableInCache = false;
-  }
-
-  // Synchronize this so not everyone's doing it at once.
-  static synchronized void createTablesIfNotExist() throws IOException {
-    if (!tablesCreated) {
-      for (String name : tableNames) {
-        if (self.get().conn.getHBaseTable(name, true) == null) {
-          List<byte[]> families = columnFamilies.get(name);
-          self.get().conn.createHBaseTable(name, families);
-        }
-      }
-      tablesCreated = true;
-    }
-  }
-
-  /**********************************************************************************************
-   * Transaction related methods
-   *********************************************************************************************/
-
-  /**
-   * Begin a transaction
-   */
-  void begin() {
-    try {
-      conn.beginTransaction();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  /**
-   * Commit a transaction
-   */
-  void commit() {
-    try {
-      conn.commitTransaction();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  void rollback() {
-    try {
-      conn.rollbackTransaction();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  void close() throws IOException {
-    conn.close();
-  }
-
-  /**********************************************************************************************
-   * Database related methods
-   *********************************************************************************************/
-
-  /**
-   * Fetch a database object
-   * @param name name of the database to fetch
-   * @return the database object, or null if there is no such database
-   * @throws IOException
-   */
-  Database getDb(String name) throws IOException {
-    byte[] key = HBaseUtils.buildKey(name);
-    byte[] serialized = read(DB_TABLE, key, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return null;
-    return HBaseUtils.deserializeDatabase(name, serialized);
-  }
-
-  /**
-   * Get a list of databases.
-   * @param regex Regular expression to use in searching for database names.  It is expected to
-   *              be a Java regular expression.  If it is null then all databases will be returned.
-   * @return list of databases matching the regular expression.
-   * @throws IOException
-   */
-  List<Database> scanDatabases(String regex) throws IOException {
-    Filter filter = null;
-    if (regex != null) {
-      filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
-    }
-    Iterator<Result> iter =
-        scan(DB_TABLE, CATALOG_CF, CATALOG_COL, filter);
-    List<Database> databases = new ArrayList<>();
-    while (iter.hasNext()) {
-      Result result = iter.next();
-      databases.add(HBaseUtils.deserializeDatabase(result.getRow(),
-          result.getValue(CATALOG_CF, CATALOG_COL)));
-    }
-    return databases;
-  }
-
-  /**
-   * Store a database object
-   * @param database database object to store
-   * @throws IOException
-   */
-  void putDb(Database database) throws IOException {
-    byte[][] serialized = HBaseUtils.serializeDatabase(database);
-    store(DB_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-  }
-
-  /**
-   * Drop a database
-   * @param name name of db to drop
-   * @throws IOException
-   */
-  void deleteDb(String name) throws IOException {
-    byte[] key = HBaseUtils.buildKey(name);
-    delete(DB_TABLE, key, null, null);
-  }
-
-  /**
-   * Print out the database. Intended for use by {@link org.apache.hadoop.hive.metastore.hbase.HBaseSchemaTool}
-   * @param name name of database to print
-   * @return string printout of database
-   */
-  String printDatabase(String name) throws IOException, TException {
-    Database db = getDb(name);
-    if (db == null) return noSuch(name, "database");
-    else return dumpThriftObject(db);
-  }
-
-  /**
-   * Print out databases.
-   * @param regex regular to use to search for databases
-   * @return databases as a string, one each
-   * @throws IOException
-   * @throws TException
-   */
-  List<String> printDatabases(String regex) throws IOException, TException {
-    List<Database> dbs = scanDatabases(regex);
-    if (dbs.size() == 0) {
-      return noMatch(regex, "database");
-    } else {
-      List<String> lines = new ArrayList<>();
-      for (Database db : dbs) lines.add(dumpThriftObject(db));
-      return lines;
-    }
-  }
-
-  int getDatabaseCount() throws IOException {
-    Filter fil = new FirstKeyOnlyFilter();
-    Iterator<Result> iter = scan(DB_TABLE, fil);
-    return Iterators.size(iter);
-  }
-
-  /**********************************************************************************************
-   * Function related methods
-   *********************************************************************************************/
-
-  /**
-   * Fetch a function object
-   * @param dbName name of the database the function is in
-   * @param functionName name of the function to fetch
-   * @return the function object, or null if there is no such function
-   * @throws IOException
-   */
-  Function getFunction(String dbName, String functionName) throws IOException {
-    byte[] key = HBaseUtils.buildKey(dbName, functionName);
-    byte[] serialized = read(FUNC_TABLE, key, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return null;
-    return HBaseUtils.deserializeFunction(dbName, functionName, serialized);
-  }
-
-  /**
-   * Get a list of functions.
-   * @param dbName Name of the database to search in.
-   * @param regex Regular expression to use in searching for function names.  It is expected to
-   *              be a Java regular expression.  If it is null then all functions will be returned.
-   * @return list of functions matching the regular expression.
-   * @throws IOException
-   */
-  List<Function> scanFunctions(String dbName, String regex) throws IOException {
-    byte[] keyPrefix = null;
-    if (dbName != null) {
-      keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName);
-    }
-    Filter filter = null;
-    if (regex != null) {
-      filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
-    }
-    Iterator<Result> iter =
-        scan(FUNC_TABLE, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix), CATALOG_CF, CATALOG_COL, filter);
-    List<Function> functions = new ArrayList<>();
-    while (iter.hasNext()) {
-      Result result = iter.next();
-      functions.add(HBaseUtils.deserializeFunction(result.getRow(),
-                                                   result.getValue(CATALOG_CF, CATALOG_COL)));
-    }
-    return functions;
-  }
-
-  /**
-   * Store a function object
-   * @param function function object to store
-   * @throws IOException
-   */
-  void putFunction(Function function) throws IOException {
-    byte[][] serialized = HBaseUtils.serializeFunction(function);
-    store(FUNC_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-  }
-
-  /**
-   * Drop a function
-   * @param dbName name of database the function is in
-   * @param functionName name of function to drop
-   * @throws IOException
-   */
-  void deleteFunction(String dbName, String functionName) throws IOException {
-    byte[] key = HBaseUtils.buildKey(dbName, functionName);
-    delete(FUNC_TABLE, key, null, null);
-  }
-
-  /**
-   * Print out a function
-   * @param key key to get the function, must include dbname.
-   * @return string of the function
-   * @throws IOException
-   * @throws TException
-   */
-  String printFunction(String key) throws IOException, TException {
-    byte[] k = HBaseUtils.buildKey(key);
-    byte[] serialized = read(FUNC_TABLE, k, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return noSuch(key, "function");
-    Function func = HBaseUtils.deserializeFunction(k, serialized);
-    return dumpThriftObject(func);
-  }
-
-  /**
-   * Print out functions
-   * @param regex regular expression to use in matching functions
-   * @return list of strings, one function each
-   * @throws IOException
-   * @throws TException
-   */
-  List<String> printFunctions(String regex) throws IOException, TException {
-    Filter  filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
-    Iterator<Result> iter = scan(FUNC_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
-    List<String> lines = new ArrayList<>();
-    while (iter.hasNext()) {
-      Result result = iter.next();
-      lines.add(dumpThriftObject(HBaseUtils.deserializeFunction(result.getRow(),
-          result.getValue(CATALOG_CF, CATALOG_COL))));
-    }
-    if (lines.size() == 0) lines = noMatch(regex, "function");
-    return lines;
-  }
-
-  /**********************************************************************************************
-   * Global privilege related methods
-   *********************************************************************************************/
-
-  /**
-   * Fetch the global privileges object
-   * @return
-   * @throws IOException
-   */
-  PrincipalPrivilegeSet getGlobalPrivs() throws IOException {
-    byte[] key = GLOBAL_PRIVS_KEY;
-    byte[] serialized = read(GLOBAL_PRIVS_TABLE, key, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return null;
-    return HBaseUtils.deserializePrincipalPrivilegeSet(serialized);
-  }
-
-  /**
-   * Store the global privileges object
-   * @throws IOException
-   */
-  void putGlobalPrivs(PrincipalPrivilegeSet privs) throws IOException {
-    byte[] key = GLOBAL_PRIVS_KEY;
-    byte[] serialized = HBaseUtils.serializePrincipalPrivilegeSet(privs);
-    store(GLOBAL_PRIVS_TABLE, key, CATALOG_CF, CATALOG_COL, serialized);
-  }
-
-  /**
-   * Print out the global privileges.
-   * @return string containing the global privileges
-   * @throws IOException
-   * @throws TException
-   */
-  String printGlobalPrivs() throws IOException, TException {
-    PrincipalPrivilegeSet pps = getGlobalPrivs();
-    if (pps == null) return "No global privileges";
-    else return dumpThriftObject(pps);
-  }
-
-  /**********************************************************************************************
-   * Partition related methods
-   *********************************************************************************************/
-
-  /**
-   * Fetch one partition
-   * @param dbName database table is in
-   * @param tableName table partition is in
-   * @param partVals list of values that specify the partition, given in the same order as the
-   *                 columns they belong to
-   * @return The partition objec,t or null if there is no such partition
-   * @throws IOException
-   */
-  Partition getPartition(String dbName, String tableName, List<String> partVals)
-      throws IOException {
-    return getPartition(dbName, tableName, partVals, true);
-  }
-
-  /**
-   * Get a set of specific partitions.  This cannot be used to do a scan, each partition must be
-   * completely specified.  This does not use the partition cache.
-   * @param dbName database table is in
-   * @param tableName table partitions are in
-   * @param partValLists list of list of values, each list should uniquely identify one partition
-   * @return a list of partition objects.
-   * @throws IOException
-   */
-   List<Partition> getPartitions(String dbName, String tableName, List<String> partTypes,
-       List<List<String>> partValLists) throws IOException {
-     List<Partition> parts = new ArrayList<>(partValLists.size());
-     List<Get> gets = new ArrayList<>(partValLists.size());
-     for (List<String> partVals : partValLists) {
-       byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName, partTypes, partVals);
-       Get get = new Get(key);
-       get.addColumn(CATALOG_CF, CATALOG_COL);
-       gets.add(get);
-     }
-     HTableInterface htab = conn.getHBaseTable(PART_TABLE);
-     Result[] results = htab.get(gets);
-     for (int i = 0; i < results.length; i++) {
-       HBaseUtils.StorageDescriptorParts sdParts =
-           HBaseUtils.deserializePartition(dbName, tableName, partValLists.get(i),
-               results[i].getValue(CATALOG_CF, CATALOG_COL));
-       StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
-       HBaseUtils.assembleStorageDescriptor(sd, sdParts);
-       parts.add(sdParts.containingPartition);
-     }
-
-     return parts;
-  }
-
-  /**
-   * Add a partition.  This should only be called for new partitions.  For altering existing
-   * partitions this should not be called as it will blindly increment the ref counter for the
-   * storage descriptor.
-   * @param partition partition object to add
-   * @throws IOException
-   */
-  void putPartition(Partition partition) throws IOException {
-    byte[] hash = putStorageDescriptor(partition.getSd());
-    byte[][] serialized = HBaseUtils.serializePartition(partition,
-        HBaseUtils.getPartitionKeyTypes(getTable(partition.getDbName(), partition.getTableName()).getPartitionKeys()), hash);
-    store(PART_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-    partCache.put(partition.getDbName(), partition.getTableName(), partition);
-  }
-
-  /**
-   * Replace an existing partition.
-   * @param oldPart partition to be replaced
-   * @param newPart partitiion to replace it with
-   * @throws IOException
-   */
-  void replacePartition(Partition oldPart, Partition newPart, List<String> partTypes) throws IOException {
-    byte[] hash;
-    byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldPart.getSd(), md);
-    byte[] newHash = HBaseUtils.hashStorageDescriptor(newPart.getSd(), md);
-    if (Arrays.equals(oldHash, newHash)) {
-      hash = oldHash;
-    } else {
-      decrementStorageDescriptorRefCount(oldPart.getSd());
-      hash = putStorageDescriptor(newPart.getSd());
-    }
-    byte[][] serialized = HBaseUtils.serializePartition(newPart,
-        HBaseUtils.getPartitionKeyTypes(getTable(newPart.getDbName(), newPart.getTableName()).getPartitionKeys()), hash);
-    store(PART_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-    partCache.put(newPart.getDbName(), newPart.getTableName(), newPart);
-    if (!oldPart.getTableName().equals(newPart.getTableName())) {
-      deletePartition(oldPart.getDbName(), oldPart.getTableName(), partTypes, oldPart.getValues());
-    }
-  }
-
-  /**
-   * Add a group of partitions.  This should only be used when all partitions are new.  It
-   * blindly increments the ref count on the storage descriptor.
-   * @param partitions list of partitions to add
-   * @throws IOException
-   */
-  void putPartitions(List<Partition> partitions) throws IOException {
-    List<Put> puts = new ArrayList<>(partitions.size());
-    for (Partition partition : partitions) {
-      byte[] hash = putStorageDescriptor(partition.getSd());
-      List<String> partTypes = HBaseUtils.getPartitionKeyTypes(
-          getTable(partition.getDbName(), partition.getTableName()).getPartitionKeys());
-      byte[][] serialized = HBaseUtils.serializePartition(partition, partTypes, hash);
-      Put p = new Put(serialized[0]);
-      p.add(CATALOG_CF, CATALOG_COL, serialized[1]);
-      puts.add(p);
-      partCache.put(partition.getDbName(), partition.getTableName(), partition);
-    }
-    HTableInterface htab = conn.getHBaseTable(PART_TABLE);
-    htab.put(puts);
-    conn.flush(htab);
-  }
-
-  void replacePartitions(List<Partition> oldParts, List<Partition> newParts, List<String> oldPartTypes) throws IOException {
-    if (oldParts.size() != newParts.size()) {
-      throw new RuntimeException("Number of old and new partitions must match.");
-    }
-    List<Put> puts = new ArrayList<>(newParts.size());
-    for (int i = 0; i < newParts.size(); i++) {
-      byte[] hash;
-      byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldParts.get(i).getSd(), md);
-      byte[] newHash = HBaseUtils.hashStorageDescriptor(newParts.get(i).getSd(), md);
-      if (Arrays.equals(oldHash, newHash)) {
-        hash = oldHash;
-      } else {
-        decrementStorageDescriptorRefCount(oldParts.get(i).getSd());
-        hash = putStorageDescriptor(newParts.get(i).getSd());
-      }
-      Partition newPart = newParts.get(i);
-      byte[][] serialized = HBaseUtils.serializePartition(newPart,
-          HBaseUtils.getPartitionKeyTypes(getTable(newPart.getDbName(), newPart.getTableName()).getPartitionKeys()), hash);
-      Put p = new Put(serialized[0]);
-      p.add(CATALOG_CF, CATALOG_COL, serialized[1]);
-      puts.add(p);
-      partCache.put(newParts.get(i).getDbName(), newParts.get(i).getTableName(), newParts.get(i));
-      if (!newParts.get(i).getTableName().equals(oldParts.get(i).getTableName())) {
-        // We need to remove the old record as well.
-        deletePartition(oldParts.get(i).getDbName(), oldParts.get(i).getTableName(), oldPartTypes,
-            oldParts.get(i).getValues(), false);
-      }
-    }
-    HTableInterface htab = conn.getHBaseTable(PART_TABLE);
-    htab.put(puts);
-    conn.flush(htab);
-  }
-
-  /**
-   * Find all the partitions in a table.
-   * @param dbName name of the database the table is in
-   * @param tableName table name
-   * @param maxPartitions max partitions to fetch.  If negative all partitions will be returned.
-   * @return List of partitions that match the criteria.
-   * @throws IOException
-   */
-  List<Partition> scanPartitionsInTable(String dbName, String tableName, int maxPartitions)
-      throws IOException {
-    if (maxPartitions < 0) maxPartitions = Integer.MAX_VALUE;
-    Collection<Partition> cached = partCache.getAllForTable(dbName, tableName);
-    if (cached != null) {
-      return maxPartitions < cached.size()
-          ? new ArrayList<>(cached).subList(0, maxPartitions)
-          : new ArrayList<>(cached);
-    }
-    byte[] keyPrefix = HBaseUtils.buildPartitionKey(dbName, tableName, new ArrayList<String>(),
-        new ArrayList<String>(), false);
-    List<Partition> parts = scanPartitionsWithFilter(dbName, tableName, keyPrefix,
-        HBaseUtils.getEndPrefix(keyPrefix), -1, null);
-    partCache.put(dbName, tableName, parts, true);
-    return maxPartitions < parts.size() ? parts.subList(0, maxPartitions) : parts;
-  }
-
-  /**
-   * Scan partitions based on partial key information.
-   * @param dbName name of database, required
-   * @param tableName name of table, required
-   * @param partVals partial specification of values.  Any values that are unknown can instead be
-   *                 a '*'.  For example, if a table had two partition columns date
-   *                 and region (in that order), and partitions ('today', 'na'), ('today', 'eu'),
-   *                 ('tomorrow', 'na'), ('tomorrow', 'eu') then passing ['today', '*'] would return
-   *                 ('today', 'na') and ('today', 'eu') while passing ['*', 'eu'] would return
-   *                 ('today', 'eu') and ('tomorrow', 'eu').  Also the list can terminate early,
-   *                 which will be the equivalent of adding '*' for all non-included values.
-   *                 I.e. ['today'] is the same as ['today', '*'].
-   * @param maxPartitions Maximum number of entries to return.
-   * @return list of partitions that match the specified information
-   * @throws IOException
-   * @throws org.apache.hadoop.hive.metastore.api.NoSuchObjectException if the table containing
-   * the partitions can't be found.
-   */
-  List<Partition> scanPartitions(String dbName, String tableName, List<String> partVals,
-                                 int maxPartitions) throws IOException, NoSuchObjectException {
-
-    PartitionScanInfo psi = scanPartitionsInternal(dbName, tableName, partVals, maxPartitions);
-    List<Partition> parts = scanPartitionsWithFilter(dbName, tableName, psi.keyPrefix,
-        psi.endKeyPrefix, maxPartitions, psi.filter);
-    partCache.put(dbName, tableName, parts, false);
-    return parts;
-  }
-
-  List<Partition> scanPartitions(String dbName, String tableName, byte[] keyStart, byte[] keyEnd,
-                                 Filter  filter, int  maxPartitions)
-      throws IOException, NoSuchObjectException {
-    byte[] startRow = keyStart;
-    byte[] endRow;
-    if (keyEnd == null || keyEnd.length == 0) {
-      // stop when current db+table entries are over
-      endRow = HBaseUtils.getEndPrefix(startRow);
-    } else {
-      endRow = keyEnd;
-    }
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Scanning partitions with start row <" + new String(startRow) + "> and end row <"
-          + new String(endRow) + ">");
-    }
-    return scanPartitionsWithFilter(dbName, tableName, startRow, endRow, maxPartitions, filter);
-  }
-
-  /**
-   * Delete a partition
-   * @param dbName database name that table is in
-   * @param tableName table partition is in
-   * @param partVals partition values that define this partition, in the same order as the
-   *                 partition columns they are values for
-   * @throws IOException
-   */
-  void deletePartition(String dbName, String tableName, List<String> partTypes,
-      List<String> partVals) throws IOException {
-    deletePartition(dbName, tableName, partTypes, partVals, true);
-  }
-
-  /**
-   * Print out a partition.
-   * @param partKey The key for the partition.  This must include dbname.tablename._partkeys_
-   *                where _partkeys_ is a dot separated list of partition values in the proper
-   *                order.
-   * @return string containing the partition
-   * @throws IOException
-   * @throws TException
-   */
-  String printPartition(String partKey) throws IOException, TException {
-    // First figure out the table and fetch it
-    String[] partKeyParts = partKey.split(HBaseUtils.KEY_SEPARATOR_STR);
-    if (partKeyParts.length < 3) return noSuch(partKey, "partition");
-    Table table = getTable(partKeyParts[0], partKeyParts[1]);
-    if (table == null) return noSuch(partKey, "partition");
-
-    byte[] key = HBaseUtils.buildPartitionKey(partKeyParts[0], partKeyParts[1],
-        HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys()),
-        Arrays.asList(Arrays.copyOfRange(partKeyParts, 2, partKeyParts.length)));
-    @SuppressWarnings("deprecation")
-    HTableInterface htab = conn.getHBaseTable(PART_TABLE);
-    Get g = new Get(key);
-    g.addColumn(CATALOG_CF, CATALOG_COL);
-    g.addFamily(STATS_CF);
-    Result result = htab.get(g);
-    if (result.isEmpty()) return noSuch(partKey, "partition");
-    return printOnePartition(result);
-  }
-
-  /**
-   * Print partitions
-   * @param partKey a partial partition key.  This must match the beginings of the partition key.
-   *                It can be just dbname.tablename, or dbname.table.pval... where pval are the
-   *                partition values in order.  They must be in the correct order and they must
-   *                be literal values (no regular expressions)
-   * @return partitions as strings
-   * @throws IOException
-   * @throws TException
-   */
-  List<String> printPartitions(String partKey) throws IOException, TException {
-    // First figure out the table and fetch it
-    // Split on dot here rather than the standard separator because this will be passed in as a
-    // regex, even though we aren't fully supporting regex's.
-    String[] partKeyParts = partKey.split("\\.");
-    if (partKeyParts.length < 2) return noMatch(partKey, "partition");
-    List<String> partVals = partKeyParts.length == 2 ? Arrays.asList("*") :
-        Arrays.asList(Arrays.copyOfRange(partKeyParts, 2, partKeyParts.length));
-    PartitionScanInfo psi;
-    try {
-      psi =
-          scanPartitionsInternal(partKeyParts[0], partKeyParts[1], partVals, -1);
-    } catch (NoSuchObjectException e) {
-      return noMatch(partKey, "partition");
-    }
-
-    @SuppressWarnings("deprecation")
-    HTableInterface htab = conn.getHBaseTable(PART_TABLE);
-    Scan scan = new Scan();
-    scan.addColumn(CATALOG_CF, CATALOG_COL);
-    scan.addFamily(STATS_CF);
-    scan.setStartRow(psi.keyPrefix);
-    scan.setStopRow(psi.endKeyPrefix);
-    scan.setFilter(psi.filter);
-    Iterator<Result> iter = htab.getScanner(scan).iterator();
-    if (!iter.hasNext()) return noMatch(partKey, "partition");
-    List<String> lines = new ArrayList<>();
-    while (iter.hasNext()) {
-      lines.add(printOnePartition(iter.next()));
-    }
-    return lines;
-  }
-
-  int getPartitionCount() throws IOException {
-    Filter fil = new FirstKeyOnlyFilter();
-    Iterator<Result> iter = scan(PART_TABLE, fil);
-    return Iterators.size(iter);
-  }
-
-  private String printOnePartition(Result result) throws IOException, TException {
-    byte[] key = result.getRow();
-    HBaseUtils.StorageDescriptorParts sdParts =
-        HBaseUtils.deserializePartition(key, result.getValue(CATALOG_CF, CATALOG_COL), this);
-    StringBuilder builder = new StringBuilder();
-    builder.append(dumpThriftObject(sdParts.containingPartition))
-        .append(" sdHash: ")
-        .append(Base64.encodeBase64URLSafeString(sdParts.sdHash))
-        .append(" stats:");
-    NavigableMap<byte[], byte[]> statsCols = result.getFamilyMap(STATS_CF);
-    for (Map.Entry<byte[], byte[]> statsCol : statsCols.entrySet()) {
-      builder.append(" column ")
-          .append(new String(statsCol.getKey(), HBaseUtils.ENCODING))
-          .append(": ");
-      ColumnStatistics pcs = buildColStats(key, false);
-      ColumnStatisticsObj cso = HBaseUtils.deserializeStatsForOneColumn(pcs, statsCol.getValue());
-      builder.append(dumpThriftObject(cso));
-    }
-    return builder.toString();
-  }
-
-  private void deletePartition(String dbName, String tableName, List<String> partTypes,
-        List<String> partVals, boolean decrementRefCnt) throws IOException {
-    // Find the partition so I can get the storage descriptor and drop it
-    partCache.remove(dbName, tableName, partVals);
-    if (decrementRefCnt) {
-      Partition p = getPartition(dbName, tableName, partVals, false);
-      decrementStorageDescriptorRefCount(p.getSd());
-    }
-    byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName, partTypes, partVals);
-    delete(PART_TABLE, key, null, null);
-  }
-
-  private Partition getPartition(String dbName, String tableName, List<String> partVals,
-                                 boolean populateCache) throws IOException {
-    Partition cached = partCache.get(dbName, tableName, partVals);
-    if (cached != null) return cached;
-    byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName,
-        HBaseUtils.getPartitionKeyTypes(getTable(dbName, tableName).getPartitionKeys()), partVals);
-    byte[] serialized = read(PART_TABLE, key, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return null;
-    HBaseUtils.StorageDescriptorParts sdParts =
-        HBaseUtils.deserializePartition(dbName, tableName, partVals, serialized);
-    StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
-    HBaseUtils.assembleStorageDescriptor(sd, sdParts);
-    if (populateCache) partCache.put(dbName, tableName, sdParts.containingPartition);
-    return sdParts.containingPartition;
-  }
-
-
-  private static class PartitionScanInfo {
-    final String dbName;
-    final String tableName;
-    final byte[] keyPrefix;
-    final byte[] endKeyPrefix;
-    final int maxPartitions;
-    final Filter filter;
-
-    PartitionScanInfo(String d, String t, byte[] k, byte[] e, int m, Filter f) {
-      dbName = d;
-      tableName = t;
-      keyPrefix = k;
-      endKeyPrefix = e;
-      maxPartitions = m;
-      filter = f;
-    }
-
-    @Override
-    public String toString() {
-      return new StringBuilder("dbName:")
-          .append(dbName)
-          .append(" tableName:")
-          .append(tableName)
-          .append(" keyPrefix:")
-          .append(Base64.encodeBase64URLSafeString(keyPrefix))
-          .append(" endKeyPrefix:")
-          .append(Base64.encodeBase64URLSafeString(endKeyPrefix))
-          .append(" maxPartitions:")
-          .append(maxPartitions)
-          .append(" filter:")
-          .append(filter.toString())
-          .toString();
-    }
-  }
-
-  private PartitionScanInfo scanPartitionsInternal(String dbName, String tableName,
-                                                   List<String> partVals, int maxPartitions)
-      throws IOException, NoSuchObjectException {
-    // First, build as much of the key as we can so that we make the scan as tight as possible.
-    List<String> keyElements = new ArrayList<>();
-    keyElements.add(dbName);
-    keyElements.add(tableName);
-
-    int firstStar = -1;
-    for (int i = 0; i < partVals.size(); i++) {
-      if ("*".equals(partVals.get(i))) {
-        firstStar = i;
-        break;
-      } else {
-        // empty string equals to null partition,
-        // means star
-        if (partVals.get(i).equals("")) {
-          break;
-        } else {
-          keyElements.add(partVals.get(i));
-        }
-      }
-    }
-
-    byte[] keyPrefix;
-    // We need to fetch the table to determine if the user fully specified the partitions or
-    // not, as it affects how we build the key.
-    Table table = getTable(dbName, tableName);
-    if (table == null) {
-      throw new NoSuchObjectException("Unable to find table " + dbName + "." + tableName);
-    }
-    keyPrefix = HBaseUtils.buildPartitionKey(dbName, tableName,
-        HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys().subList(0, keyElements.size()-2)),
-        keyElements.subList(2, keyElements.size()));
-
-    // Now, build a filter out of the remaining keys
-    List<PartitionKeyComparator.Range> ranges = new ArrayList<PartitionKeyComparator.Range>();
-    List<Operator> ops = new ArrayList<Operator>();
-    if (!(partVals.size() == table.getPartitionKeys().size() && firstStar == -1)) {
-
-      for (int i = Math.max(0, firstStar);
-           i < table.getPartitionKeys().size() && i < partVals.size(); i++) {
-
-        if ("*".equals(partVals.get(i))) {
-          PartitionKeyComparator.Operator op = new PartitionKeyComparator.Operator(
-              PartitionKeyComparator.Operator.Type.LIKE,
-              table.getPartitionKeys().get(i).getName(),
-              ".*");
-          ops.add(op);
-        } else {
-          PartitionKeyComparator.Range range = new PartitionKeyComparator.Range(
-              table.getPartitionKeys().get(i).getName(),
-              new PartitionKeyComparator.Mark(partVals.get(i), true),
-              new PartitionKeyComparator.Mark(partVals.get(i), true));
-          ranges.add(range);
-        }
-      }
-    }
-
-    Filter filter = null;
-    if (!ranges.isEmpty() || !ops.isEmpty()) {
-      filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new PartitionKeyComparator(
-          StringUtils.join(HBaseUtils.getPartitionNames(table.getPartitionKeys()), ","),
-          StringUtils.join(HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys()), ","),
-          ranges, ops));
-    }
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Scanning partitions with prefix <" + new String(keyPrefix) + "> and filter <" +
-          filter + ">");
-    }
-
-    return new PartitionScanInfo(dbName, tableName, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix),
-        maxPartitions, filter);
-  }
-
-  private List<Partition> scanPartitionsWithFilter(String dbName, String tableName,
-                                                   byte[] startRow, byte [] endRow, int maxResults,
-                                                   Filter filter) throws IOException {
-    Iterator<Result> iter =
-        scan(PART_TABLE, startRow, endRow, CATALOG_CF, CATALOG_COL, filter);
-    List<FieldSchema> tablePartitions = getTable(dbName, tableName).getPartitionKeys();
-    List<Partition> parts = new ArrayList<>();
-    int numToFetch = maxResults < 0 ? Integer.MAX_VALUE : maxResults;
-    for (int i = 0; i < numToFetch && iter.hasNext(); i++) {
-      Result result = iter.next();
-      HBaseUtils.StorageDescriptorParts sdParts = HBaseUtils.deserializePartition(dbName, tableName,
-          tablePartitions, result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL), conf);
-      StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
-      HBaseUtils.assembleStorageDescriptor(sd, sdParts);
-      parts.add(sdParts.containingPartition);
-    }
-    return parts;
-  }
-
-  /**********************************************************************************************
-   * Role related methods
-   *********************************************************************************************/
-
-  /**
-   * Fetch the list of all roles for a user
-   * @param userName name of the user
-   * @return the list of all roles this user participates in
-   * @throws IOException
-   */
-  List<String> getUserRoles(String userName) throws IOException {
-    byte[] key = HBaseUtils.buildKey(userName);
-    byte[] serialized = read(USER_TO_ROLE_TABLE, key, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return null;
-    return HBaseUtils.deserializeRoleList(serialized);
-  }
-
-  /**
-   * Find all roles directly participated in by a given principal.  This builds the role cache
-   * because it assumes that subsequent calls may be made to find roles participated in indirectly.
-   * @param name username or role name
-   * @param type user or role
-   * @return map of role name to grant info for all roles directly participated in.
-   */
-  List<Role> getPrincipalDirectRoles(String name, PrincipalType type)
-      throws IOException {
-    buildRoleCache();
-
-    Set<String> rolesFound = new HashSet<>();
-    for (Map.Entry<String, HbaseMetastoreProto.RoleGrantInfoList> e : roleCache.entrySet()) {
-      for (HbaseMetastoreProto.RoleGrantInfo giw : e.getValue().getGrantInfoList()) {
-        if (HBaseUtils.convertPrincipalTypes(giw.getPrincipalType()) == type &&
-            giw.getPrincipalName().equals(name)) {
-          rolesFound.add(e.getKey());
-          break;
-        }
-      }
-    }
-    List<Role> directRoles = new ArrayList<>(rolesFound.size());
-    List<Get> gets = new ArrayList<>();
-    HTableInterface htab = conn.getHBaseTable(ROLE_TABLE);
-    for (String roleFound : rolesFound) {
-      byte[] key = HBaseUtils.buildKey(roleFound);
-      Get g = new Get(key);
-      g.addColumn(CATALOG_CF, CATALOG_COL);
-      gets.add(g);
-    }
-
-    Result[] results = htab.get(gets);
-    for (int i = 0; i < results.length; i++) {
-      byte[] serialized = results[i].getValue(CATALOG_CF, CATALOG_COL);
-      if (serialized != null) {
-        directRoles.add(HBaseUtils.deserializeRole(results[i].getRow(), serialized));
-      }
-    }
-
-    return directRoles;
-  }
-
-  /**
-   * Fetch all roles and users included directly in a given role.
-   * @param roleName name of the principal
-   * @return a list of all roles included in this role
-   * @throws IOException
-   */
-  HbaseMetastoreProto.RoleGrantInfoList getRolePrincipals(String roleName)
-      throws IOException, NoSuchObjectException {
-    HbaseMetastoreProto.RoleGrantInfoList rolePrincipals = roleCache.get(roleName);
-    if (rolePrincipals != null) return rolePrincipals;
-    byte[] key = HBaseUtils.buildKey(roleName);
-    byte[] serialized = read(ROLE_TABLE, key, CATALOG_CF, ROLES_COL);
-    if (serialized == null) return null;
-    rolePrincipals = HbaseMetastoreProto.RoleGrantInfoList.parseFrom(serialized);
-    roleCache.put(roleName, rolePrincipals);
-    return rolePrincipals;
-  }
-
-  /**
-   * Given a role, find all users who are either directly or indirectly participate in this role.
-   * This is expensive, it should be used sparingly.  It scan the entire userToRole table and
-   * does a linear search on each entry.
-   * @param roleName name of the role
-   * @return set of all users in the role
-   * @throws IOException
-   */
-  Set<String> findAllUsersInRole(String roleName) throws IOException {
-    // Walk the userToRole table and collect every user that matches this role.
-    Set<String> users = new HashSet<>();
-    Iterator<Result> iter = scan(USER_TO_ROLE_TABLE, CATALOG_CF, CATALOG_COL);
-    while (iter.hasNext()) {
-      Result result = iter.next();
-      List<String> roleList =
-          HBaseUtils.deserializeRoleList(result.getValue(CATALOG_CF, CATALOG_COL));
-      for (String rn : roleList) {
-        if (rn.equals(roleName)) {
-          users.add(new String(result.getRow(), HBaseUtils.ENCODING));
-          break;
-        }
-      }
-    }
-    return users;
-  }
-
-  /**
-   * Add a principal to a role.
-   * @param roleName name of the role to add principal to
-   * @param grantInfo grant information for this principal.
-   * @throws java.io.IOException
-   * @throws NoSuchObjectException
-   *
-   */
-  void addPrincipalToRole(String roleName, HbaseMetastoreProto.RoleGrantInfo grantInfo)
-      throws IOException, NoSuchObjectException {
-    HbaseMetastoreProto.RoleGrantInfoList proto = getRolePrincipals(roleName);
-    List<HbaseMetastoreProto.RoleGrantInfo> rolePrincipals = new ArrayList<>();
-    if (proto != null) {
-      rolePrincipals.addAll(proto.getGrantInfoList());
-    }
-
-    rolePrincipals.add(grantInfo);
-    proto = HbaseMetastoreProto.RoleGrantInfoList.newBuilder()
-        .addAllGrantInfo(rolePrincipals)
-        .build();
-    byte[] key = HBaseUtils.buildKey(roleName);
-    store(ROLE_TABLE, key, CATALOG_CF, ROLES_COL, proto.toByteArray());
-    roleCache.put(roleName, proto);
-  }
-
-  /**
-   * Drop a principal from a role.
-   * @param roleName Name of the role to drop the principal from
-   * @param principalName name of the principal to drop from the role
-   * @param type user or role
-   * @param grantOnly if this is true, just remove the grant option, don't actually remove the
-   *                  user from the role.
-   * @throws NoSuchObjectException
-   * @throws IOException
-   */
-  void dropPrincipalFromRole(String roleName, String principalName, PrincipalType type,
-                             boolean grantOnly)
-      throws NoSuchObjectException, IOException {
-    HbaseMetastoreProto.RoleGrantInfoList proto = getRolePrincipals(roleName);
-    if (proto == null) return;
-    List<HbaseMetastoreProto.RoleGrantInfo> rolePrincipals = new ArrayList<>();
-    rolePrincipals.addAll(proto.getGrantInfoList());
-
-    for (int i = 0; i < rolePrincipals.size(); i++) {
-      if (HBaseUtils.convertPrincipalTypes(rolePrincipals.get(i).getPrincipalType()) == type &&
-          rolePrincipals.get(i).getPrincipalName().equals(principalName)) {
-        if (grantOnly) {
-          rolePrincipals.set(i,
-              HbaseMetastoreProto.RoleGrantInfo.newBuilder(rolePrincipals.get(i))
-                  .setGrantOption(false)
-                  .build());
-        } else {
-          rolePrincipals.remove(i);
-        }
-        break;
-      }
-    }
-    byte[] key = HBaseUtils.buildKey(roleName);
-    proto = HbaseMetastoreProto.RoleGrantInfoList.newBuilder()
-        .addAllGrantInfo(rolePrincipals)
-        .build();
-    store(ROLE_TABLE, key, CATALOG_CF, ROLES_COL, proto.toByteArray());
-    roleCache.put(roleName, proto);
-  }
-
-  /**
-   * Rebuild the row for a given user in the USER_TO_ROLE table.  This is expensive.  It
-   * should be called as infrequently as possible.
-   * @param userName name of the user
-   * @throws IOException
-   */
-  void buildRoleMapForUser(String userName) throws IOException, NoSuchObjectException {
-    // This is mega ugly.  Hopefully we don't have to do this too often.
-    // First, scan the role table and put it all in memory
-    buildRoleCache();
-    LOG.debug("Building role map for " + userName);
-
-    // Second, find every role the user participates in directly.
-    Set<String> rolesToAdd = new HashSet<>();
-    Set<String> rolesToCheckNext = new HashSet<>();
-    for (Map.Entry<String, HbaseMetastoreProto.RoleGrantInfoList> e : roleCache.entrySet()) {
-      for (HbaseMetastoreProto.RoleGrantInfo grantInfo : e.getValue().getGrantInfoList()) {
-        if (HBaseUtils.convertPrincipalTypes(grantInfo.getPrincipalType()) == PrincipalType.USER &&
-            userName .equals(grantInfo.getPrincipalName())) {
-          rolesToAdd.add(e.getKey());
-          rolesToCheckNext.add(e.getKey());
-          LOG.debug("Adding " + e.getKey() + " to list of roles user is in directly");
-          break;
-        }
-      }
-    }
-
-    // Third, find every role the user participates in indirectly (that is, they have been
-    // granted into role X and role Y has been granted into role X).
-    while (rolesToCheckNext.size() > 0) {
-      Set<String> tmpRolesToCheckNext = new HashSet<>();
-      for (String roleName : rolesToCheckNext) {
-        HbaseMetastoreProto.RoleGrantInfoList grantInfos = roleCache.get(roleName);
-        if (grantInfos == null) continue;  // happens when a role contains no grants
-        for (HbaseMetastoreProto.RoleGrantInfo grantInfo : grantInfos.getGrantInfoList()) {
-          if (HBaseUtils.convertPrincipalTypes(grantInfo.getPrincipalType()) == PrincipalType.ROLE &&
-              rolesToAdd.add(grantInfo.getPrincipalName())) {
-            tmpRolesToCheckNext.add(grantInfo.getPrincipalName());
-            LOG.debug("Adding " + grantInfo.getPrincipalName() +
-                " to list of roles user is in indirectly");
-          }
-        }
-      }
-      rolesToCheckNext = tmpRolesToCheckNext;
-    }
-
-    byte[] key = HBaseUtils.buildKey(userName);
-    byte[] serialized = HBaseUtils.serializeRoleList(new ArrayList<>(rolesToAdd));
-    store(USER_TO_ROLE_TABLE, key, CATALOG_CF, CATALOG_COL, serialized);
-  }
-
-  /**
-   * Remove all of the grants for a role.  This is not cheap.
-   * @param roleName Role to remove from all other roles and grants
-   * @throws IOException
-   */
-  void removeRoleGrants(String roleName) throws IOException {
-    buildRoleCache();
-
-    List<Put> puts = new ArrayList<>();
-    // First, walk the role table and remove any references to this role
-    for (Map.Entry<String, HbaseMetastoreProto.RoleGrantInfoList> e : roleCache.entrySet()) {
-      boolean madeAChange = false;
-      List<HbaseMetastoreProto.RoleGrantInfo> rgil = new ArrayList<>();
-      rgil.addAll(e.getValue().getGrantInfoList());
-      for (int i = 0; i < rgil.size(); i++) {
-        if (HBaseUtils.convertPrincipalTypes(rgil.get(i).getPrincipalType()) == PrincipalType.ROLE &&
-            rgil.get(i).getPrincipalName().equals(roleName)) {
-          rgil.remove(i);
-          madeAChange = true;
-          break;
-        }
-      }
-      if (madeAChange) {
-        Put put = new Put(HBaseUtils.buildKey(e.getKey()));
-        HbaseMetastoreProto.RoleGrantInfoList proto =
-            HbaseMetastoreProto.RoleGrantInfoList.newBuilder()
-            .addAllGrantInfo(rgil)
-            .build();
-        put.add(CATALOG_CF, ROLES_COL, proto.toByteArray());
-        puts.add(put);
-        roleCache.put(e.getKey(), proto);
-      }
-    }
-
-    if (puts.size() > 0) {
-      HTableInterface htab = conn.getHBaseTable(ROLE_TABLE);
-      htab.put(puts);
-      conn.flush(htab);
-    }
-
-    // Remove any global privileges held by this role
-    PrincipalPrivilegeSet global = getGlobalPrivs();
-    if (global != null &&
-        global.getRolePrivileges() != null &&
-        global.getRolePrivileges().remove(roleName) != null) {
-      putGlobalPrivs(global);
-    }
-
-    // Now, walk the db table
-    puts.clear();
-    List<Database> dbs = scanDatabases(null);
-    if (dbs == null) dbs = new ArrayList<>(); // rare, but can happen
-    for (Database db : dbs) {
-      if (db.getPrivileges() != null &&
-          db.getPrivileges().getRolePrivileges() != null &&
-          db.getPrivileges().getRolePrivileges().remove(roleName) != null) {
-        byte[][] serialized = HBaseUtils.serializeDatabase(db);
-        Put put = new Put(serialized[0]);
-        put.add(CATALOG_CF, CATALOG_COL, serialized[1]);
-        puts.add(put);
-      }
-    }
-
-    if (puts.size() > 0) {
-      HTableInterface htab = conn.getHBaseTable(DB_TABLE);
-      htab.put(puts);
-      conn.flush(htab);
-    }
-
-    // Finally, walk the table table
-    puts.clear();
-    for (Database db : dbs) {
-      List<Table> tables = scanTables(db.getName(), null);
-      if (tables != null) {
-        for (Table table : tables) {
-          if (table.getPrivileges() != null &&
-              table.getPrivileges().getRolePrivileges() != null &&
-              table.getPrivileges().getRolePrivileges().remove(roleName) != null) {
-            byte[][] serialized = HBaseUtils.serializeTable(table,
-                HBaseUtils.hashStorageDescriptor(table.getSd(), md));
-            Put put = new Put(serialized[0]);
-            put.add(CATALOG_CF, CATALOG_COL, serialized[1]);
-            puts.add(put);
-          }
-        }
-      }
-    }
-
-    if (puts.size() > 0) {
-      HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
-      htab.put(puts);
-      conn.flush(htab);
-    }
-  }
-
-  /**
-   * Fetch a role
-   * @param roleName name of the role
-   * @return role object, or null if no such role
-   * @throws IOException
-   */
-  Role getRole(String roleName) throws IOException {
-    byte[] key = HBaseUtils.buildKey(roleName);
-    byte[] serialized = read(ROLE_TABLE, key, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return null;
-    return HBaseUtils.deserializeRole(roleName, serialized);
-  }
-
-  /**
-   * Get a list of roles.
-   * @return list of all known roles.
-   * @throws IOException
-   */
-  List<Role> scanRoles() throws IOException {
-    return scanRoles(null);
-  }
-
-  /**
-   * Add a new role
-   * @param role role object
-   * @throws IOException
-   */
-  void putRole(Role role) throws IOException {
-    byte[][] serialized = HBaseUtils.serializeRole(role);
-    store(ROLE_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-  }
-
-  /**
-   * Drop a role
-   * @param roleName name of role to drop
-   * @throws IOException
-   */
-  void deleteRole(String roleName) throws IOException {
-    byte[] key = HBaseUtils.buildKey(roleName);
-    delete(ROLE_TABLE, key, null, null);
-    roleCache.remove(roleName);
-  }
-
-  String printRolesForUser(String userName) throws IOException {
-    List<String> roles = getUserRoles(userName);
-    if (roles == null || roles.size() == 0) return noSuch(userName, "user");
-    return org.apache.commons.lang.StringUtils.join(roles, ',');
-  }
-
-  List<String> printRolesForUsers(String regex) throws IOException {
-    Filter  filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
-    Iterator<Result> iter = scan(USER_TO_ROLE_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
-    List<String> lines = new ArrayList<>();
-    while (iter.hasNext()) {
-      Result result = iter.next();
-      lines.add(new String(result.getRow(), HBaseUtils.ENCODING) + ": " +
-          org.apache.commons.lang.StringUtils.join(
-            HBaseUtils.deserializeRoleList(result.getValue(CATALOG_CF, CATALOG_COL)), ','));
-    }
-    if (lines.size() == 0) lines = noMatch(regex, "user");
-    return lines;
-  }
-
-  /**
-   * Print out a role
-   * @param name name of role to print
-   * @return string printout of role
-   */
-  String printRole(String name) throws IOException, TException {
-    Role role = getRole(name);
-    if (role == null) return noSuch(name, "role");
-    else return dumpThriftObject(role);
-  }
-
-  /**
-   * Print out roles.
-   * @param regex regular to use to search for roles
-   * @return string printout of roles
-   * @throws IOException
-   * @throws TException
-   */
-  List<String> printRoles(String regex) throws IOException, TException {
-    List<Role> roles = scanRoles(regex);
-    if (roles.size() == 0) {
-      return noMatch(regex, "role");
-    } else {
-      List<String> lines = new ArrayList<>();
-      for (Role role : roles) lines.add(dumpThriftObject(role));
-      return lines;
-    }
-  }
-
-  private List<Role> scanRoles(String regex) throws IOException {
-    Filter filter = null;
-    if (regex != null) {
-      filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
-    }
-    Iterator<Result> iter = scan(ROLE_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
-    List<Role> roles = new ArrayList<>();
-    while (iter.hasNext()) {
-      Result result = iter.next();
-      roles.add(HBaseUtils.deserializeRole(result.getRow(),
-          result.getValue(CATALOG_CF, CATALOG_COL)));
-    }
-    return roles;
-  }
-
-  private void buildRoleCache() throws IOException {
-    if (!entireRoleTableInCache) {
-      Iterator<Result> roles = scan(ROLE_TABLE, CATALOG_CF, ROLES_COL);
-      while (roles.hasNext()) {
-        Result res = roles.next();
-        String roleName = new String(res.getRow(), HBaseUtils.ENCODING);
-        HbaseMetastoreProto.RoleGrantInfoList grantInfos =
-            HbaseMetastoreProto.RoleGrantInfoList.parseFrom(res.getValue(CATALOG_CF, ROLES_COL));
-        roleCache.put(roleName, grantInfos);
-      }
-      entireRoleTableInCache = true;
-    }
-  }
-
-  /**********************************************************************************************
-   * Table related methods
-   *********************************************************************************************/
-
-  /**
-   * Fetch a table object
-   * @param dbName database the table is in
-   * @param tableName table name
-   * @return Table object, or null if no such table
-   * @throws IOException
-   */
-  Table getTable(String dbName, String tableName) throws IOException {
-    return getTable(dbName, tableName, true);
-  }
-
-  /**
-   * Fetch a list of table objects.
-   * @param dbName Database that all fetched tables are in
-   * @param tableNames list of table names
-   * @return list of tables, in the same order as the provided names.
-   * @throws IOException
-   */
-  List<Table> getTables(String dbName, List<String> tableNames) throws IOException {
-    // I could implement getTable in terms of this method.  But it is such a core function
-    // that I don't want to slow it down for the much less common fetching of multiple tables.
-    List<Table> results = new ArrayList<>(tableNames.size());
-    ObjectPair<String, String>[] hashKeys = new ObjectPair[tableNames.size()];
-    boolean atLeastOneMissing = false;
-    for (int i = 0; i < tableNames.size(); i++) {
-      hashKeys[i] = new ObjectPair<>(dbName, tableNames.get(i));
-      // The result may be null, but we still want to add it so that we have a slot in the list
-      // for it.
-      results.add(tableCache.get(hashKeys[i]));
-      if (results.get(i) == null) atLeastOneMissing = true;
-    }
-    if (!atLeastOneMissing) return results;
-
-    // Now build a single get that will fetch the remaining tables
-    List<Get> gets = new ArrayList<>();
-    HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
-    for (int i = 0; i < tableNames.size(); i++) {
-      if (results.get(i) != null) continue;
-      byte[] key = HBaseUtils.buildKey(dbName, tableNames.get(i));
-      Get g = new Get(key);
-      g.addColumn(CATALOG_CF, CATALOG_COL);
-      gets.add(g);
-    }
-    Result[] res = htab.get(gets);
-    for (int i = 0, nextGet = 0; i < tableNames.size(); i++) {
-      if (results.get(i) != null) continue;
-      byte[] serialized = res[nextGet++].getValue(CATALOG_CF, CATALOG_COL);
-      if (serialized != null) {
-        HBaseUtils.StorageDescriptorParts sdParts =
-            HBaseUtils.deserializeTable(dbName, tableNames.get(i), serialized);
-        StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
-        HBaseUtils.assembleStorageDescriptor(sd, sdParts);
-        tableCache.put(hashKeys[i], sdParts.containingTable);
-        results.set(i, sdParts.containingTable);
-      }
-    }
-    return results;
-  }
-
-  /**
-   * Get a list of tables.
-   * @param dbName Database these tables are in
-   * @param regex Regular expression to use in searching for table names.  It is expected to
-   *              be a Java regular expression.  If it is null then all tables in the indicated
-   *              database will be returned.
-   * @return list of tables matching the regular expression.
-   * @throws IOException
-   */
-  List<Table> scanTables(String dbName, String regex) throws IOException {
-    // There's no way to know whether all the tables we are looking for are
-    // in the cache, so we would need to scan one way or another.  Thus there's no value in hitting
-    // the cache for this function.
-    byte[] keyPrefix = null;
-    if (dbName != null) {
-      keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName);
-    }
-    Filter filter = null;
-    if (regex != null) {
-      filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
-    }
-    Iterator<Result> iter =
-        scan(TABLE_TABLE, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix),
-            CATALOG_CF, CATALOG_COL, filter);
-    List<Table> tables = new ArrayList<>();
-    while (iter.hasNext()) {
-      Result result = iter.next();
-      HBaseUtils.StorageDescriptorParts sdParts =
-          HBaseUtils.deserializeTable(result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL));
-      StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
-      HBaseUtils.assembleStorageDescriptor(sd, sdParts);
-      tables.add(sdParts.containingTable);
-    }
-    return tables;
-  }
-
-  /**
-   * Put a table object.  This should only be called when the table is new (create table) as it
-   * will blindly add/increment the storage descriptor.  If you are altering an existing table
-   * call {@link #replaceTable} instead.
-   * @param table table object
-   * @throws IOException
-   */
-  void putTable(Table table) throws IOException {
-    byte[] hash = putStorageDescriptor(table.getSd());
-    byte[][] serialized = HBaseUtils.serializeTable(table, hash);
-    store(TABLE_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-    tableCache.put(new ObjectPair<>(table.getDbName(), table.getTableName()), table);
-  }
-
-  /**
-   * Replace an existing table.  This will also compare the storage descriptors and see if the
-   * reference count needs to be adjusted
-   * @param oldTable old version of the table
-   * @param newTable new version of the table
-   */
-  void replaceTable(Table oldTable, Table newTable) throws IOException {
-    byte[] hash;
-    byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldTable.getSd(), md);
-    byte[] newHash = HBaseUtils.hashStorageDescriptor(newTable.getSd(), md);
-    if (Arrays.equals(oldHash, newHash)) {
-      hash = oldHash;
-    } else {
-      decrementStorageDescriptorRefCount(oldTable.getSd());
-      hash = putStorageDescriptor(newTable.getSd());
-    }
-    byte[][] serialized = HBaseUtils.serializeTable(newTable, hash);
-    store(TABLE_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-    tableCache.put(new ObjectPair<>(newTable.getDbName(), newTable.getTableName()), newTable);
-    if (!oldTable.getTableName().equals(newTable.getTableName())) {
-      deleteTable(oldTable.getDbName(), oldTable.getTableName());
-    }
-  }
-
-  /**
-   * Delete a table
-   * @param dbName name of database table is in
-   * @param tableName table to drop
-   * @throws IOException
-   */
-  void deleteTable(String dbName, String tableName) throws IOException {
-    deleteTable(dbName, tableName, true);
-  }
-
-  /**
-   * Print out a table.
-   * @param name The name for the table.  This must include dbname.tablename
-   * @return string containing the table
-   * @throws IOException
-   * @throws TException
-   */
-  String printTable(String name) throws IOException, TException {
-    byte[] key = HBaseUtils.buildKey(name);
-    @SuppressWarnings("deprecation")
-    HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
-    Get g = new Get(key);
-    g.addColumn(CATALOG_CF, CATALOG_COL);
-    g.addFamily(STATS_CF);
-    Result result = htab.get(g);
-    if (result.isEmpty()) return noSuch(name, "table");
-    return printOneTable(result);
-  }
-
-  /**
-   * Print tables
-   * @param regex to use to find the tables.  Remember that dbname is in each
-   *              table name.
-   * @return tables as strings
-   * @throws IOException
-   * @throws TException
-   */
-  List<String> printTables(String regex) throws IOException, TException {
-    Filter  filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
-    @SuppressWarnings("deprecation")
-    HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
-    Scan scan = new Scan();
-    scan.addColumn(CATALOG_CF, CATALOG_COL);
-    scan.addFamily(STATS_CF);
-    scan.setFilter(filter);
-    Iterator<Result> iter = htab.getScanner(scan).iterator();
-    if (!iter.hasNext()) return noMatch(regex, "table");
-    List<String> lines = new ArrayList<>();
-    while (iter.hasNext()) {
-      lines.add(printOneTable(iter.next()));
-    }
-    return lines;
-  }
-
-  int getTableCount() throws IOException {
-    Filter fil = new FirstKeyOnlyFilter();
-    Iterator<Result> iter = scan(TABLE_TABLE, fil);
-    return Iterators.size(iter);
-  }
-
-  private String printOneTable(Result result) throws IOException, TException {
-    byte[] key = result.getRow();
-    HBaseUtils.StorageDescriptorParts sdParts =
-        HBaseUtils.deserializeTable(key, result.getValue(CATALOG_CF, CATALOG_COL));
-    StringBuilder builder = new StringBuilder();
-    builder.append(dumpThriftObject(sdParts.containingTable))
-        .append(" sdHash: ")
-        .append(Base64.encodeBase64URLSafeString(sdParts.sdHash))
-        .append(" stats:");
-    NavigableMap<byte[], byte[]> statsCols = result.getFamilyMap(STATS_CF);
-    for (Map.Entry<byte[], byte[]> statsCol : statsCols.entrySet()) {
-      builder.append(" column ")
-          .append(new String(statsCol.getKey(), HBaseUtils.ENCODING))
-          .append(": ");
-      ColumnStatistics pcs = buildColStats(key, true);
-      ColumnStatisticsObj cso = HBaseUtils.deserializeStatsForOneColumn(pcs, statsCol.getValue());
-      builder.append(dumpThriftObject(cso));
-    }
-    // Add the primary key
-    List<SQLPrimaryKey> pk = getPrimaryKey(sdParts.containingTable.getDbName(),
-        sdParts.containingTable.getTableName());
-    if (pk != null && pk.size() > 0) {
-      builder.append(" primary key: ");
-      for (SQLPrimaryKey pkcol : pk) builder.append(dumpThriftObject(pkcol));
-    }
-
-    // Add any foreign keys
-    List<SQLForeignKey> fks = getForeignKeys(sdParts.containingTable.getDbName(),
-        sdParts.containingTable.getTableName());
-    if (fks != null && fks.size() > 0) {
-      builder.append(" foreign keys: ");
-      for (SQLForeignKey fkcol : fks) builder.append(dumpThriftObject(fkcol));
-
-    }
-    return builder.toString();
-  }
-
-  private void deleteTable(String dbName, String tableName, boolean decrementRefCnt)
-      throws IOException {
-    tableCache.remove(new ObjectPair<>(dbName, tableName));
-    if (decrementRefCnt) {
-      // Find the table so I can get the storage descriptor and drop it
-      Table t = getTable(dbName, tableName, false);
-      decrementStorageDescriptorRefCount(t.getSd());
-    }
-    byte[] key = HBaseUtils.buildKey(dbName, tableName);
-    delete(TABLE_TABLE, key, null, null);
-  }
-
-  private Table getTable(String dbName, String tableName, boolean populateCache)
-      throws IOException {
-    ObjectPair<String, String> hashKey = new ObjectPair<>(dbName, tableName);
-    Table cached = tableCache.get(hashKey);
-    if (cached != null) return cached;
-    byte[] key = HBaseUtils.buildKey(dbName, tableName);
-    byte[] serialized = read(TABLE_TABLE, key, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return null;
-    HBaseUtils.StorageDescriptorParts sdParts =
-        HBaseUtils.deserializeTable(dbName, tableName, serialized);
-    StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
-    HBaseUtils.assembleStorageDescriptor(sd, sdParts);
-    if (populateCache) tableCache.put(hashKey, sdParts.containingTable);
-    return sdParts.containingTable;
-  }
-
-  /**********************************************************************************************
-   * Index related methods
-   *********************************************************************************************/
-
-  /**
-   * Put an index object.  This should only be called when the index is new (create index) as it
-   * will blindly add/increment the storage descriptor.  If you are altering an existing index
-   * call {@link #replaceIndex} instead.
-   * @param index index object
-   * @throws IOException
-   */
-  void putIndex(Index index) throws IOException {
-    byte[] hash = putStorageDescriptor(index.getSd());
-    byte[][] serialized = HBaseUtils.serializeIndex(index, hash);
-    store(INDEX_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-  }
-
-  /**
-   * Fetch an index object
-   * @param dbName database the table is in
-   * @param origTableName original table name
-   * @param indexName index name
-   * @return Index object, or null if no such table
-   * @throws IOException
-   */
-  Index getIndex(String dbName, String origTableName, String indexName) throws IOException {
-    byte[] key = HBaseUtils.buildKey(dbName, origTableName, indexName);
-    byte[] serialized = read(INDEX_TABLE, key, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return null;
-    HBaseUtils.StorageDescriptorParts sdParts =
-        HBaseUtils.deserializeIndex(dbName, origTableName, indexName, serialized);
-    StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
-    HBaseUtils.assembleStorageDescriptor(sd, sdParts);
-    return sdParts.containingIndex;
-  }
-
-  /**
-   * Delete a table
-   * @param dbName name of database table is in
-   * @param origTableName table the index is built on
-   * @param indexName index name
-   * @throws IOException
-   */
-  void deleteIndex(String dbName, String origTableName, String indexName) throws IOException {
-    deleteIndex(dbName, origTableName, indexName, true);
-  }
-
-  void deleteIndex(String dbName, String origTableName, String indexName, boolean decrementRefCnt)
-      throws IOException {
-    // Find the index so I can get the storage descriptor and drop it
-    if (decrementRefCnt) {
-      Index index = getIndex(dbName, origTableName, indexName);
-      decrementStorageDescriptorRefCount(index.getSd());
-    }
-    byte[] key = HBaseUtils.buildKey(dbName, origTableName, indexName);
-    delete(INDEX_TABLE, key, null, null);
-  }
-
-  /**
-   * Get a list of tables.
-   * @param dbName Database these tables are in
-   * @param origTableName original table name
-   * @param maxResults max indexes to fetch.  If negative all indexes will be returned.
-   * @return list of indexes of the table
-   * @throws IOException
-   */
-  List<Index> scanIndexes(String dbName, String origTableName, int maxResults) throws IOException {
-    // There's no way to know whether all the tables we are looking for are
-    // in the cache, so we would need to scan one way or another.  Thus there's no value in hitting
-    // the cache for this function.
-    byte[] keyPrefix = null;
-    if (dbName != null) {
-      keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName, origTableName);
-    }
-    Iterator<Result> iter = scan(INDEX_TABLE, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix),
-        CATALOG_CF, CATALOG_COL, null);
-    List<Index> indexes = new ArrayList<>();
-    int numToFetch = maxResults < 0 ? Integer.MAX_VALUE : maxResults;
-    for (int i = 0; i < numToFetch && iter.hasNext(); i++) {
-      Result result = iter.next();
-      HBaseUtils.StorageDescriptorParts sdParts = HBaseUtils.deserializeIndex(result.getRow(),
-          result.getValue(CATALOG_CF, CATALOG_COL));
-      StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
-      HBaseUtils.assembleStorageDescriptor(sd, sdParts);
-      indexes.add(sdParts.containingIndex);
-    }
-    return indexes;
-  }
-
-  /**
-   * Replace an existing index.  This will also compare the storage descriptors and see if the
-   * reference count needs to be adjusted
-   * @param oldIndex old version of the index
-   * @param newIndex new version of the index
-   */
-  void replaceIndex(Index oldIndex, Index newIndex) throws IOException {
-    byte[] hash;
-    byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldIndex.getSd(), md);
-    byte[] newHash = HBaseUtils.hashStorageDescriptor(newIndex.getSd(), md);
-    if (Arrays.equals(oldHash, newHash)) {
-      hash = oldHash;
-    } else {
-      decrementStorageDescriptorRefCount(oldIndex.getSd());
-      hash = putStorageDescriptor(newIndex.getSd());
-    }
-    byte[][] serialized = HBaseUtils.serializeIndex(newIndex, hash);
-    store(INDEX_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
-    if (!(oldIndex.getDbName().equals(newIndex.getDbName()) &&
-        oldIndex.getOrigTableName().equals(newIndex.getOrigTableName()) &&
-        oldIndex.getIndexName().equals(newIndex.getIndexName()))) {
-      deleteIndex(oldIndex.getDbName(), oldIndex.getOrigTableName(), oldIndex.getIndexName(), false);
-    }
-  }
-  /**********************************************************************************************
-   * StorageDescriptor related methods
-   *********************************************************************************************/
-
-  /**
-   * If this serde has already been read, then return it from the cache.  If not, read it, then
-   * return it.
-   * @param hash hash of the storage descriptor to read
-   * @return the storage descriptor
-   * @throws IOException
-   */
-  StorageDescriptor getStorageDescriptor(byte[] hash) throws IOException {
-    ByteArrayWrapper hashKey = new ByteArrayWrapper(hash);
-    StorageDescriptor cached = sdCache.get(hashKey);
-    if (cached != null) return cached;
-    LOG.debug("Not found in cache, looking in hbase");
-    byte[] serialized = read(SD_TABLE, hash, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) {
-      throw new RuntimeException("Woh, bad!  Trying to fetch a non-existent storage descriptor " +
-          "from hash " + Base64.encodeBase64String(hash));
-    }
-    StorageDescriptor sd = HBaseUtils.deserializeStorageDescriptor(serialized);
-    sdCache.put(hashKey, sd);
-    return sd;
-  }
-
-  /**
-   * Lower the reference count on the storage descriptor by one.  If it goes to zero, then it
-   * will be deleted.
-   * @param sd Storage descriptor
-   * @throws IOException
-   */
-  void decrementStorageDescriptorRefCount(StorageDescriptor sd) throws IOException {
-    byte[] key = HBaseUtils.hashStorageDescriptor(sd, md);
-    byte[] serializedRefCnt = read(SD_TABLE, key, CATALOG_CF, REF_COUNT_COL);
-    if (serializedRefCnt == null) {
-      // Someone deleted it before we got to it, no worries
-      return;
-    }
-    int refCnt = Integer.parseInt(new String(serializedRefCnt, HBaseUtils.ENCODING));
-    HTableInterface htab = conn.getHBaseTable(SD_TABLE);
-    if (--refCnt < 1) {
-      Delete d = new Delete(key);
-      // We don't use checkAndDelete here because it isn't compatible with the transaction
-      // managers.  If the transaction managers are doing their jobs then we should not need it
-      // anyway.
-      htab.delete(d);
-      sdCache.remove(new ByteArrayWrapper(key));
-    } else {
-      Put p = new Put(key);
-      p.add(CATALOG_CF, REF_COUNT_COL, Integer.toString(refCnt).getBytes(HBaseUtils.ENCODING));
-      htab.put(p);
-      conn.flush(htab);
-    }
-  }
-
-  /**
-   * Place the common parts of a storage descriptor into the cache and write the storage
-   * descriptor out to HBase.  This should only be called if you are sure that the storage
-   * descriptor needs to be added.  If you have changed a table or partition but not it's storage
-   * descriptor do not call this method, as it will increment the reference count of the storage
-   * descriptor.
-   * @param storageDescriptor storage descriptor to store.
-   * @return id of the entry in the cache, to be written in for the storage descriptor
-   */
-  byte[] putStorageDescriptor(StorageDescriptor storageDescriptor) throws IOException {
-    byte[] sd = HBaseUtils.serializeStorageDescriptor(storageDescriptor);
-    byte[] key = HBaseUtils.hashStorageDescriptor(storageDescriptor, md);
-    byte[] serializedRefCnt = read(SD_TABLE, key, CATALOG_CF, REF_COUNT_COL);
-    HTableInterface htab = conn.getHBaseTable(SD_TABLE);
-    if (serializedRefCnt == null) {
-      // We are the first to put it in the DB
-      Put p = new Put(key);
-      p.add(CATALOG_CF, CATALOG_COL, sd);
-      p.add(CATALOG_CF, REF_COUNT_COL, "1".getBytes(HBaseUtils.ENCODING));
-      htab.put(p);
-      sdCache.put(new ByteArrayWrapper(key), storageDescriptor);
-    } else {
-      // Just increment the reference count
-      int refCnt = Integer.parseInt(new String(serializedRefCnt, HBaseUtils.ENCODING)) + 1;
-      Put p = new Put(key);
-      p.add(CATALOG_CF, REF_COUNT_COL, Integer.toString(refCnt).getBytes(HBaseUtils.ENCODING));
-      htab.put(p);
-    }
-    conn.flush(htab);
-    return key;
-  }
-
-  /**
-   * Print out a storage descriptor.
-   * @param hash hash that is the key of the storage descriptor
-   * @return string version of the storage descriptor
-   */
-  String printStorageDescriptor(byte[] hash) throws IOException, TException {
-    byte[] serialized = read(SD_TABLE, hash, CATALOG_CF, CATALOG_COL);
-    if (serialized == null) return noSuch(Base64.encodeBase64URLSafeString(hash), "storage descriptor");
-    return dumpThriftObject(HBaseUtils.deserializeStorageDescriptor(serialized));
-  }
-
-  /**
-   * Print all of the storage descriptors.  This doesn't take a regular expression since the key
-   * is an md5 hash and it's hard to see how a regex on this would be useful.
-   * @return list of all storage descriptors as strings
-   * @throws IOException
-   * @throws TException
-   */
-  List<String> printStorageDescriptors() throws IOException, TException {
-    Iterator<Result> results = scan(SD_TABLE, CATALOG_CF, CATALOG_COL);
-    if (!results.hasNext()) return Arrays.asList("No storage descriptors");
-    List<String> lines = new ArrayList<>();
-    while (results.hasNext()) {
-      Result result = results.next();
-      lines.add(Base64.encodeBase64URLSafeString(result.getRow()) + ": " +
-        dumpThriftObject(HBaseUtils.deserializeStorageDescriptor(result.getValue(CATALOG_CF,
-            CATALOG_COL))));
-    }
-    return lines;
-  }
-
-  private static class ByteArrayWrapper {
-    byte[] wrapped;
-
-    ByteArrayWrapper(byte[] b) {
-      wrapped = b;
-    }
-
-    @Override
-    public boolean equals(Object other) {
-      if (other instanceof ByteArrayWrapper) {
-        return Arrays.equals(((ByteArrayWrapper)other).wrapped, wrapped);
-      } else {
-        return false;
-      }
-    }
-
-    @Override
-    public int hashCode() {
-      return Arrays.hashCode(wrapped);
-    }
-  }
-
-  /**********************************************************************************************
-   * Statistics related methods
-   *********************************************************************************************/
-
-  /**
-   * Update statistics for one or more columns for a table or a partition.
-   *
-   * @param dbName database the table is in
-   * @param tableName table to update statistics for
-   * @param partVals partition values that define partition to update statistics for. If this is
-   *          null, then these will be assumed to be table level statistics
-   * @param stats Stats object with stats for one or more columns
-   * @throws IOException
-   */
-  void updateStatistics(String dbName, String tableName, List<String> partVals,
-      ColumnStatistics stats) throws IOException {
-    byte[] key = getStatisticsKey(dbName, tableName, partVals);
-    String hbaseTable = getStatisticsTable(partVals);
-    byte[][] colnames = new byte[stats.getStatsObjSize()][];
-    byte[][] serialized = new byte[stats.getStatsObjSize()][];
-    for (int i = 0; i < stats.getStatsObjSize(); i++) {
-      ColumnStatisticsObj obj = stats.getStatsObj().get(i);
-      serialized[i] = HBaseUtils.serializeStatsForOneColumn(stats, obj);
-      String colname = obj.getColName();
-      colnames[i] = HBaseUtils.buildKey(colname);
-    }
-    store(hbaseTable, key, STATS_CF, colnames, serialized);
-  }
-
-  /**
-   * Get statistics for a table
-   *
-   * @param dbName name of database table is in
-   * @param tblName name of table
-   * @param colNames list of column names to get statistics for
-   * @return column statistics for indicated table
-   * @throws IOException
-   */
-  ColumnStatistics getTableStatistics(String dbName, String tblName, List<String> colNames)
-      throws IOException {
-    byte[] tabKey = HBaseUtils.buildKey(dbName, tblName);
-    ColumnStatistics tableStats = new ColumnStatistics();
-    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
-    statsDesc.setIsTblLevel(true);
-    statsDesc.setDbName(dbName);
-    statsDesc.setTableName(tblName);
-    tableStats.setStatsDesc(statsDesc);
-    byte[][] colKeys = new byte[colNames.size()][];
-    for (int i = 0; i < colKeys.length; i++) {
-      colKeys[i] = HBaseUtils.buildKey(colNames.get(i));
-    }
-    Result result = read(TABLE_TABLE, tabKey, STATS_CF, colKeys);
-    for (int i = 0; i < colKeys.length; i++) {
-      byte[] serializedColStats = result.getValue(STATS_CF, colKeys[i]);
-      if (serializedColStats == null) {
-        // There were no stats for this column, so skip it
-        continue;
-      }
-      ColumnStatisticsObj obj =
-          HBaseUtils.deserializeStatsForOneColumn(tableStats, serializedColStats);
-      obj.setColName(colNames.get(i));
-      tableStats.addToStatsObj(obj);
-    }
-    return tableStats;
-  }
-
-  /**
-   * Get statistics for a set of partitions
-   *
-   * @param dbName name of database table is in
-   * @param tblName table partitions are in
-   * @param partNames names of the partitions, used only to set values inside the return stats
-   *          objects
-   * @param partVals partition values for each partition, needed because this class doesn't know how
-   *          to translate from partName to partVals
-   * @param colNames column names to fetch stats for. These columns will be fetched for all
-   *          requested partitions
-   * @return list of ColumnStats, one for each partition for which we found at least one column's
-   * stats.
-   * @throws IOException
-   */
-  List<ColumnStatistics> getPartitionStatistics(String dbName, String tblName,
-      List<String> partNames, List<List<String>> partVals, List<String> colNames)
-      throws IOExcept

<TRUNCATED>

[07/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
deleted file mode 100644
index 6b7eb9e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
+++ /dev/null
@@ -1,1781 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.ResourceType;
-import org.apache.hadoop.hive.metastore.api.ResourceUri;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.ByteStream.Output;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
-import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDeWithEndPrefix;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hive.common.util.BloomFilter;
-import org.apache.hive.common.util.HiveStringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- * Utility functions
- */
-public class HBaseUtils {
-
-  final static Charset ENCODING = StandardCharsets.UTF_8;
-  final static char KEY_SEPARATOR = '\u0001';
-  final static String KEY_SEPARATOR_STR = new String(new char[] {KEY_SEPARATOR});
-
-  static final private Logger LOG = LoggerFactory.getLogger(HBaseUtils.class.getName());
-
-  /**
-   * Build a key for an object in hbase
-   * @param components
-   * @return
-   */
-  static byte[] buildKey(String... components) {
-    return buildKey(false, components);
-  }
-
-  static byte[] buildKeyWithTrailingSeparator(String... components) {
-    return buildKey(true, components);
-  }
-
-  private static byte[] buildKey(boolean trailingSeparator, String... components) {
-    String protoKey = StringUtils.join(components, KEY_SEPARATOR);
-    if (trailingSeparator) protoKey += KEY_SEPARATOR;
-    return protoKey.getBytes(ENCODING);
-  }
-
-  private static HbaseMetastoreProto.Parameters buildParameters(Map<String, String> params) {
-    List<HbaseMetastoreProto.ParameterEntry> entries = new ArrayList<>();
-    for (Map.Entry<String, String> e : params.entrySet()) {
-      entries.add(
-          HbaseMetastoreProto.ParameterEntry.newBuilder()
-              .setKey(e.getKey())
-              .setValue(e.getValue())
-              .build());
-    }
-    return HbaseMetastoreProto.Parameters.newBuilder()
-        .addAllParameter(entries)
-        .build();
-  }
-
-  private static Map<String, String> buildParameters(HbaseMetastoreProto.Parameters protoParams) {
-    Map<String, String> params = new HashMap<>();
-    for (HbaseMetastoreProto.ParameterEntry pe : protoParams.getParameterList()) {
-      params.put(pe.getKey(), pe.getValue());
-    }
-    return params;
-  }
-
-
-  private static List<HbaseMetastoreProto.PrincipalPrivilegeSetEntry>
-  buildPrincipalPrivilegeSetEntry(Map<String, List<PrivilegeGrantInfo>> entries) {
-    List<HbaseMetastoreProto.PrincipalPrivilegeSetEntry> results = new ArrayList<>();
-    for (Map.Entry<String, List<PrivilegeGrantInfo>> entry : entries.entrySet()) {
-      results.add(HbaseMetastoreProto.PrincipalPrivilegeSetEntry.newBuilder()
-          .setPrincipalName(entry.getKey())
-          .addAllPrivileges(buildPrivilegeGrantInfo(entry.getValue()))
-          .build());
-    }
-    return results;
-  }
-
-  private static List<HbaseMetastoreProto.PrivilegeGrantInfo> buildPrivilegeGrantInfo(
-      List<PrivilegeGrantInfo> privileges) {
-    List<HbaseMetastoreProto.PrivilegeGrantInfo> results = new ArrayList<>();
-    for (PrivilegeGrantInfo privilege : privileges) {
-      HbaseMetastoreProto.PrivilegeGrantInfo.Builder builder =
-          HbaseMetastoreProto.PrivilegeGrantInfo.newBuilder();
-      if (privilege.getPrivilege() != null) builder.setPrivilege(privilege.getPrivilege());
-      builder.setCreateTime(privilege.getCreateTime());
-      if (privilege.getGrantor() != null) builder.setGrantor(privilege.getGrantor());
-      if (privilege.getGrantorType() != null) {
-        builder.setGrantorType(convertPrincipalTypes(privilege.getGrantorType()));
-      }
-      builder.setGrantOption(privilege.isGrantOption());
-      results.add(builder.build());
-    }
-    return results;
-  }
-
-  /**
-   * Convert Thrift.PrincipalType to HbaseMetastoreProto.principalType
-   * @param type
-   * @return
-   */
-  static HbaseMetastoreProto.PrincipalType convertPrincipalTypes(PrincipalType type) {
-    switch (type) {
-      case USER: return HbaseMetastoreProto.PrincipalType.USER;
-      case ROLE: return HbaseMetastoreProto.PrincipalType.ROLE;
-      default: throw new RuntimeException("Unknown principal type " + type.toString());
-    }
-  }
-
-  /**
-   * Convert principalType from HbaseMetastoreProto to Thrift.PrincipalType
-   * @param type
-   * @return
-   */
-  static PrincipalType convertPrincipalTypes(HbaseMetastoreProto.PrincipalType type) {
-    switch (type) {
-      case USER: return PrincipalType.USER;
-      case ROLE: return PrincipalType.ROLE;
-      default: throw new RuntimeException("Unknown principal type " + type.toString());
-    }
-  }
-
-  private static Map<String, List<PrivilegeGrantInfo>> convertPrincipalPrivilegeSetEntries(
-      List<HbaseMetastoreProto.PrincipalPrivilegeSetEntry> entries) {
-    Map<String, List<PrivilegeGrantInfo>> map = new HashMap<>();
-    for (HbaseMetastoreProto.PrincipalPrivilegeSetEntry entry : entries) {
-      map.put(entry.getPrincipalName(), convertPrivilegeGrantInfos(entry.getPrivilegesList()));
-    }
-    return map;
-  }
-
-  private static List<PrivilegeGrantInfo> convertPrivilegeGrantInfos(
-      List<HbaseMetastoreProto.PrivilegeGrantInfo> privileges) {
-    List<PrivilegeGrantInfo> results = new ArrayList<>();
-    for (HbaseMetastoreProto.PrivilegeGrantInfo proto : privileges) {
-      PrivilegeGrantInfo pgi = new PrivilegeGrantInfo();
-      if (proto.hasPrivilege()) pgi.setPrivilege(proto.getPrivilege());
-      pgi.setCreateTime((int)proto.getCreateTime());
-      if (proto.hasGrantor()) pgi.setGrantor(proto.getGrantor());
-      if (proto.hasGrantorType()) {
-        pgi.setGrantorType(convertPrincipalTypes(proto.getGrantorType()));
-      }
-      if (proto.hasGrantOption()) pgi.setGrantOption(proto.getGrantOption());
-      results.add(pgi);
-    }
-    return results;
-  }
-
-  private static HbaseMetastoreProto.PrincipalPrivilegeSet
-  buildPrincipalPrivilegeSet(PrincipalPrivilegeSet pps) {
-    HbaseMetastoreProto.PrincipalPrivilegeSet.Builder builder =
-        HbaseMetastoreProto.PrincipalPrivilegeSet.newBuilder();
-    if (pps.getUserPrivileges() != null) {
-      builder.addAllUsers(buildPrincipalPrivilegeSetEntry(pps.getUserPrivileges()));
-    }
-    if (pps.getRolePrivileges() != null) {
-      builder.addAllRoles(buildPrincipalPrivilegeSetEntry(pps.getRolePrivileges()));
-    }
-    return builder.build();
-  }
-
-  private static PrincipalPrivilegeSet buildPrincipalPrivilegeSet(
-      HbaseMetastoreProto.PrincipalPrivilegeSet proto) throws InvalidProtocolBufferException {
-    PrincipalPrivilegeSet pps = null;
-    if (!proto.getUsersList().isEmpty() || !proto.getRolesList().isEmpty()) {
-      pps = new PrincipalPrivilegeSet();
-      if (!proto.getUsersList().isEmpty()) {
-        pps.setUserPrivileges(convertPrincipalPrivilegeSetEntries(proto.getUsersList()));
-      }
-      if (!proto.getRolesList().isEmpty()) {
-        pps.setRolePrivileges(convertPrincipalPrivilegeSetEntries(proto.getRolesList()));
-      }
-    }
-    return pps;
-  }
-  /**
-   * Serialize a PrincipalPrivilegeSet
-   * @param pps
-   * @return
-   */
-  static byte[] serializePrincipalPrivilegeSet(PrincipalPrivilegeSet pps) {
-    return buildPrincipalPrivilegeSet(pps).toByteArray();
-  }
-
-  /**
-   * Deserialize a PrincipalPrivilegeSet
-   * @param serialized
-   * @return
-   * @throws InvalidProtocolBufferException
-   */
-  static PrincipalPrivilegeSet deserializePrincipalPrivilegeSet(byte[] serialized)
-      throws InvalidProtocolBufferException {
-    HbaseMetastoreProto.PrincipalPrivilegeSet proto =
-        HbaseMetastoreProto.PrincipalPrivilegeSet.parseFrom(serialized);
-    return buildPrincipalPrivilegeSet(proto);
-  }
-
-  /**
-   * Serialize a role
-   * @param role
-   * @return two byte arrays, first contains the key, the second the serialized value.
-   */
-  static byte[][] serializeRole(Role role) {
-    byte[][] result = new byte[2][];
-    result[0] = buildKey(role.getRoleName());
-    HbaseMetastoreProto.Role.Builder builder = HbaseMetastoreProto.Role.newBuilder();
-    builder.setCreateTime(role.getCreateTime());
-    if (role.getOwnerName() != null) builder.setOwnerName(role.getOwnerName());
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-  /**
-   * Deserialize a role.  This method should be used when the rolename is already known as it
-   * doesn't have to re-deserialize it.
-   * @param roleName name of the role
-   * @param value value fetched from hbase
-   * @return A role
-   * @throws InvalidProtocolBufferException
-   */
-  static Role deserializeRole(String roleName, byte[] value)
-      throws InvalidProtocolBufferException {
-    Role role = new Role();
-    role.setRoleName(roleName);
-    HbaseMetastoreProto.Role protoRole =
-        HbaseMetastoreProto.Role.parseFrom(value);
-    role.setCreateTime((int)protoRole.getCreateTime());
-    if (protoRole.hasOwnerName()) role.setOwnerName(protoRole.getOwnerName());
-    return role;
-  }
-
-  /**
-   * Deserialize a role.  This method should be used when the rolename is not already known (eg
-   * when doing a scan).
-   * @param key key from hbase
-   * @param value value from hbase
-   * @return a role
-   * @throws InvalidProtocolBufferException
-   */
-  static Role deserializeRole(byte[] key, byte[] value)
-      throws InvalidProtocolBufferException {
-    String roleName = new String(key, ENCODING);
-    return deserializeRole(roleName, value);
-  }
-
-  /**
-   * Serialize a list of role names
-   * @param roles
-   * @return
-   */
-  static byte[] serializeRoleList(List<String> roles) {
-    return HbaseMetastoreProto.RoleList.newBuilder()
-        .addAllRole(roles)
-        .build()
-        .toByteArray();
-  }
-
-  static List<String> deserializeRoleList(byte[] value) throws InvalidProtocolBufferException {
-    HbaseMetastoreProto.RoleList proto = HbaseMetastoreProto.RoleList.parseFrom(value);
-    return new ArrayList<>(proto.getRoleList());
-  }
-
-  /**
-   * Serialize a database
-   * @param db
-   * @return two byte arrays, first contains the key, the second the serialized value.
-   */
-  static byte[][] serializeDatabase(Database db) {
-    byte[][] result = new byte[2][];
-    result[0] = buildKey(HiveStringUtils.normalizeIdentifier(db.getName()));
-    HbaseMetastoreProto.Database.Builder builder = HbaseMetastoreProto.Database.newBuilder();
-
-    if (db.getDescription() != null) builder.setDescription(db.getDescription());
-    if (db.getLocationUri() != null) builder.setUri(db.getLocationUri());
-    if (db.getParameters() != null) builder.setParameters(buildParameters(db.getParameters()));
-    if (db.getPrivileges() != null) {
-      builder.setPrivileges(buildPrincipalPrivilegeSet(db.getPrivileges()));
-    }
-    if (db.getOwnerName() != null) builder.setOwnerName(db.getOwnerName());
-    if (db.getOwnerType() != null) builder.setOwnerType(convertPrincipalTypes(db.getOwnerType()));
-
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-
-  /**
-   * Deserialize a database.  This method should be used when the db anme is already known as it
-   * doesn't have to re-deserialize it.
-   * @param dbName name of the role
-   * @param value value fetched from hbase
-   * @return A database
-   * @throws InvalidProtocolBufferException
-   */
-  static Database deserializeDatabase(String dbName, byte[] value)
-      throws InvalidProtocolBufferException {
-    Database db = new Database();
-    db.setName(dbName);
-    HbaseMetastoreProto.Database protoDb = HbaseMetastoreProto.Database.parseFrom(value);
-    if (protoDb.hasDescription()) db.setDescription(protoDb.getDescription());
-    if (protoDb.hasUri()) db.setLocationUri(protoDb.getUri());
-    if (protoDb.hasParameters()) db.setParameters(buildParameters(protoDb.getParameters()));
-    if (protoDb.hasPrivileges()) {
-      db.setPrivileges(buildPrincipalPrivilegeSet(protoDb.getPrivileges()));
-    }
-    if (protoDb.hasOwnerName()) db.setOwnerName(protoDb.getOwnerName());
-    if (protoDb.hasOwnerType()) db.setOwnerType(convertPrincipalTypes(protoDb.getOwnerType()));
-
-    return db;
-  }
-
-  /**
-   * Deserialize a database.  This method should be used when the db name is not already known (eg
-   * when doing a scan).
-   * @param key key from hbase
-   * @param value value from hbase
-   * @return a role
-   * @throws InvalidProtocolBufferException
-   */
-  static Database deserializeDatabase(byte[] key, byte[] value)
-      throws InvalidProtocolBufferException {
-    String dbName = new String(key, ENCODING);
-    return deserializeDatabase(dbName, value);
-  }
-
-  /**
-   * Serialize a function
-   * @param func function to serialize
-   * @return two byte arrays, first contains the key, the second the value.
-   */
-  static byte[][] serializeFunction(Function func) {
-    byte[][] result = new byte[2][];
-    result[0] = buildKey(func.getDbName(), func.getFunctionName());
-    HbaseMetastoreProto.Function.Builder builder = HbaseMetastoreProto.Function.newBuilder();
-    if (func.getClassName() != null) builder.setClassName(func.getClassName());
-    if (func.getOwnerName() != null) builder.setOwnerName(func.getOwnerName());
-    if (func.getOwnerType() != null) {
-      builder.setOwnerType(convertPrincipalTypes(func.getOwnerType()));
-    }
-    builder.setCreateTime(func.getCreateTime());
-    if (func.getFunctionType() != null) {
-      builder.setFunctionType(convertFunctionTypes(func.getFunctionType()));
-    }
-    if (func.getResourceUris() != null) {
-      for (ResourceUri uri : func.getResourceUris()) {
-        builder.addResourceUris(HbaseMetastoreProto.Function.ResourceUri.newBuilder()
-            .setResourceType(convertResourceTypes(uri.getResourceType()))
-            .setUri(uri.getUri()));
-      }
-    }
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-  /**
-   * Deserialize a function.  This method should be used when the function and db name are
-   * already known.
-   * @param dbName name of the database the function is in
-   * @param functionName name of the function
-   * @param value serialized value of the function
-   * @return function as an object
-   * @throws InvalidProtocolBufferException
-   */
-  static Function deserializeFunction(String dbName, String functionName, byte[] value)
-      throws InvalidProtocolBufferException {
-    Function func = new Function();
-    func.setDbName(dbName);
-    func.setFunctionName(functionName);
-    HbaseMetastoreProto.Function protoFunc = HbaseMetastoreProto.Function.parseFrom(value);
-    if (protoFunc.hasClassName()) func.setClassName(protoFunc.getClassName());
-    if (protoFunc.hasOwnerName()) func.setOwnerName(protoFunc.getOwnerName());
-    if (protoFunc.hasOwnerType()) {
-      func.setOwnerType(convertPrincipalTypes(protoFunc.getOwnerType()));
-    }
-    func.setCreateTime((int)protoFunc.getCreateTime());
-    if (protoFunc.hasFunctionType()) {
-      func.setFunctionType(convertFunctionTypes(protoFunc.getFunctionType()));
-    }
-    for (HbaseMetastoreProto.Function.ResourceUri protoUri : protoFunc.getResourceUrisList()) {
-      func.addToResourceUris(new ResourceUri(convertResourceTypes(protoUri.getResourceType()),
-                                             protoUri.getUri()));
-    }
-    return func;
-  }
-
-  /**
-   * Deserialize a function.  This method should be used when the dbname and function name are
-   * not already known, such as in a scan.
-   * @param key key from hbase
-   * @param value value from hbase
-   * @return function object
-   * @throws InvalidProtocolBufferException
-   */
-  static Function deserializeFunction(byte[] key, byte[] value)
-      throws  InvalidProtocolBufferException {
-    String[] keys = deserializeKey(key);
-    return deserializeFunction(keys[0], keys[1], value);
-  }
-
-  private static HbaseMetastoreProto.Function.FunctionType convertFunctionTypes(FunctionType type) {
-    switch (type) {
-    case JAVA: return HbaseMetastoreProto.Function.FunctionType.JAVA;
-    default: throw new RuntimeException("Unknown function type " + type.toString());
-    }
-  }
-
-  private static FunctionType convertFunctionTypes(HbaseMetastoreProto.Function.FunctionType type) {
-    switch (type) {
-    case JAVA: return FunctionType.JAVA;
-    default: throw new RuntimeException("Unknown function type " + type.toString());
-    }
-  }
-
-  private static HbaseMetastoreProto.Function.ResourceUri.ResourceType
-  convertResourceTypes(ResourceType type) {
-    switch (type) {
-    case JAR: return HbaseMetastoreProto.Function.ResourceUri.ResourceType.JAR;
-    case FILE: return HbaseMetastoreProto.Function.ResourceUri.ResourceType.FILE;
-    case ARCHIVE: return HbaseMetastoreProto.Function.ResourceUri.ResourceType.ARCHIVE;
-    default: throw new RuntimeException("Unknown resource type " + type.toString());
-    }
-  }
-
-  private static ResourceType convertResourceTypes(
-      HbaseMetastoreProto.Function.ResourceUri.ResourceType type) {
-    switch (type) {
-    case JAR: return ResourceType.JAR;
-    case FILE: return ResourceType.FILE;
-    case ARCHIVE: return ResourceType.ARCHIVE;
-    default: throw new RuntimeException("Unknown resource type " + type.toString());
-    }
-  }
-
-  private static List<FieldSchema>
-  convertFieldSchemaListFromProto(List<HbaseMetastoreProto.FieldSchema> protoList) {
-    List<FieldSchema> schemas = new ArrayList<>(protoList.size());
-    for (HbaseMetastoreProto.FieldSchema proto : protoList) {
-      schemas.add(new FieldSchema(proto.getName(), proto.getType(),
-          proto.hasComment() ? proto.getComment() : null));
-    }
-    return schemas;
-  }
-
-  private static List<HbaseMetastoreProto.FieldSchema>
-  convertFieldSchemaListToProto(List<FieldSchema> schemas) {
-    List<HbaseMetastoreProto.FieldSchema> protoList = new ArrayList<>(schemas.size());
-    for (FieldSchema fs : schemas) {
-      HbaseMetastoreProto.FieldSchema.Builder builder =
-          HbaseMetastoreProto.FieldSchema.newBuilder();
-      builder
-          .setName(fs.getName())
-          .setType(fs.getType());
-      if (fs.getComment() != null) builder.setComment(fs.getComment());
-      protoList.add(builder.build());
-    }
-    return protoList;
-  }
-
-  /**
-   * Serialize a storage descriptor.
-   * @param sd storage descriptor to serialize
-   * @return serialized storage descriptor.
-   */
-  static byte[] serializeStorageDescriptor(StorageDescriptor sd)  {
-    HbaseMetastoreProto.StorageDescriptor.Builder builder =
-        HbaseMetastoreProto.StorageDescriptor.newBuilder();
-    builder.addAllCols(convertFieldSchemaListToProto(sd.getCols()));
-    if (sd.getInputFormat() != null) {
-      builder.setInputFormat(sd.getInputFormat());
-    }
-    if (sd.getOutputFormat() != null) {
-      builder.setOutputFormat(sd.getOutputFormat());
-    }
-    builder.setIsCompressed(sd.isCompressed());
-    builder.setNumBuckets(sd.getNumBuckets());
-    if (sd.getSerdeInfo() != null) {
-      HbaseMetastoreProto.StorageDescriptor.SerDeInfo.Builder serdeBuilder =
-          HbaseMetastoreProto.StorageDescriptor.SerDeInfo.newBuilder();
-      SerDeInfo serde = sd.getSerdeInfo();
-      if (serde.getName() != null) {
-        serdeBuilder.setName(serde.getName());
-      }
-      if (serde.getSerializationLib() != null) {
-        serdeBuilder.setSerializationLib(serde.getSerializationLib());
-      }
-      if (serde.getParameters() != null) {
-        serdeBuilder.setParameters(buildParameters(serde.getParameters()));
-      }
-      builder.setSerdeInfo(serdeBuilder);
-    }
-    if (sd.getBucketCols() != null) {
-      builder.addAllBucketCols(sd.getBucketCols());
-    }
-    if (sd.getSortCols() != null) {
-      List<Order> orders = sd.getSortCols();
-      List<HbaseMetastoreProto.StorageDescriptor.Order> protoList = new ArrayList<>(orders.size());
-      for (Order order : orders) {
-        protoList.add(HbaseMetastoreProto.StorageDescriptor.Order.newBuilder()
-            .setColumnName(order.getCol())
-            .setOrder(order.getOrder())
-            .build());
-      }
-      builder.addAllSortCols(protoList);
-    }
-    if (sd.getSkewedInfo() != null) {
-      HbaseMetastoreProto.StorageDescriptor.SkewedInfo.Builder skewBuilder =
-          HbaseMetastoreProto.StorageDescriptor.SkewedInfo.newBuilder();
-      SkewedInfo skewed = sd.getSkewedInfo();
-      if (skewed.getSkewedColNames() != null) {
-        skewBuilder.addAllSkewedColNames(skewed.getSkewedColNames());
-      }
-      if (skewed.getSkewedColValues() != null) {
-        for (List<String> innerList : skewed.getSkewedColValues()) {
-          HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueList.Builder listBuilder =
-              HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueList.newBuilder();
-          listBuilder.addAllSkewedColValue(innerList);
-          skewBuilder.addSkewedColValues(listBuilder);
-        }
-      }
-      if (skewed.getSkewedColValueLocationMaps() != null) {
-        for (Map.Entry<List<String>, String> e : skewed.getSkewedColValueLocationMaps().entrySet()) {
-          HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueLocationMap.Builder mapBuilder =
-              HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueLocationMap.newBuilder();
-          mapBuilder.addAllKey(e.getKey());
-          mapBuilder.setValue(e.getValue());
-          skewBuilder.addSkewedColValueLocationMaps(mapBuilder);
-        }
-      }
-      builder.setSkewedInfo(skewBuilder);
-    }
-    builder.setStoredAsSubDirectories(sd.isStoredAsSubDirectories());
-
-    return builder.build().toByteArray();
-  }
-
-  /**
-   * Produce a hash for the storage descriptor
-   * @param sd storage descriptor to hash
-   * @param md message descriptor to use to generate the hash
-   * @return the hash as a byte array
-   */
-  public static byte[] hashStorageDescriptor(StorageDescriptor sd, MessageDigest md)  {
-    // Note all maps and lists have to be absolutely sorted.  Otherwise we'll produce different
-    // results for hashes based on the OS or JVM being used.
-    md.reset();
-    for (FieldSchema fs : sd.getCols()) {
-      md.update(fs.getName().getBytes(ENCODING));
-      md.update(fs.getType().getBytes(ENCODING));
-      if (fs.getComment() != null) md.update(fs.getComment().getBytes(ENCODING));
-    }
-    if (sd.getInputFormat() != null) {
-      md.update(sd.getInputFormat().getBytes(ENCODING));
-    }
-    if (sd.getOutputFormat() != null) {
-      md.update(sd.getOutputFormat().getBytes(ENCODING));
-    }
-    md.update(sd.isCompressed() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
-    md.update(Integer.toString(sd.getNumBuckets()).getBytes(ENCODING));
-    if (sd.getSerdeInfo() != null) {
-      SerDeInfo serde = sd.getSerdeInfo();
-      if (serde.getName() != null) {
-        md.update(serde.getName().getBytes(ENCODING));
-      }
-      if (serde.getSerializationLib() != null) {
-        md.update(serde.getSerializationLib().getBytes(ENCODING));
-      }
-      if (serde.getParameters() != null) {
-        SortedMap<String, String> params = new TreeMap<>(serde.getParameters());
-        for (Map.Entry<String, String> param : params.entrySet()) {
-          md.update(param.getKey().getBytes(ENCODING));
-          md.update(param.getValue().getBytes(ENCODING));
-        }
-      }
-    }
-    if (sd.getBucketCols() != null) {
-      List<String> bucketCols = new ArrayList<>(sd.getBucketCols());
-      for (String bucket : bucketCols) md.update(bucket.getBytes(ENCODING));
-    }
-    if (sd.getSortCols() != null) {
-      SortedSet<Order> orders = new TreeSet<>(sd.getSortCols());
-      for (Order order : orders) {
-        md.update(order.getCol().getBytes(ENCODING));
-        md.update(Integer.toString(order.getOrder()).getBytes(ENCODING));
-      }
-    }
-    if (sd.getSkewedInfo() != null) {
-      SkewedInfo skewed = sd.getSkewedInfo();
-      if (skewed.getSkewedColNames() != null) {
-        SortedSet<String> colnames = new TreeSet<>(skewed.getSkewedColNames());
-        for (String colname : colnames) md.update(colname.getBytes(ENCODING));
-      }
-      if (skewed.getSkewedColValues() != null) {
-        SortedSet<String> sortedOuterList = new TreeSet<>();
-        for (List<String> innerList : skewed.getSkewedColValues()) {
-          SortedSet<String> sortedInnerList = new TreeSet<>(innerList);
-          sortedOuterList.add(StringUtils.join(sortedInnerList, "."));
-        }
-        for (String colval : sortedOuterList) md.update(colval.getBytes(ENCODING));
-      }
-      if (skewed.getSkewedColValueLocationMaps() != null) {
-        SortedMap<String, String> sortedMap = new TreeMap<>();
-        for (Map.Entry<List<String>, String> smap : skewed.getSkewedColValueLocationMaps().entrySet()) {
-          SortedSet<String> sortedKey = new TreeSet<>(smap.getKey());
-          sortedMap.put(StringUtils.join(sortedKey, "."), smap.getValue());
-        }
-        for (Map.Entry<String, String> e : sortedMap.entrySet()) {
-          md.update(e.getKey().getBytes(ENCODING));
-          md.update(e.getValue().getBytes(ENCODING));
-        }
-      }
-      md.update(sd.isStoredAsSubDirectories() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
-    }
-
-    return md.digest();
-  }
-
-  static StorageDescriptor deserializeStorageDescriptor(byte[] serialized)
-      throws InvalidProtocolBufferException {
-    HbaseMetastoreProto.StorageDescriptor proto =
-        HbaseMetastoreProto.StorageDescriptor.parseFrom(serialized);
-    StorageDescriptor sd = new StorageDescriptor();
-    sd.setCols(convertFieldSchemaListFromProto(proto.getColsList()));
-    if (proto.hasInputFormat()) sd.setInputFormat(proto.getInputFormat());
-    if (proto.hasOutputFormat()) sd.setOutputFormat(proto.getOutputFormat());
-    sd.setCompressed(proto.getIsCompressed());
-    sd.setNumBuckets(proto.getNumBuckets());
-    if (proto.hasSerdeInfo()) {
-      SerDeInfo serde = new SerDeInfo();
-      serde.setName(proto.getSerdeInfo().hasName()?
-          proto.getSerdeInfo().getName():null);
-      serde.setSerializationLib(proto.getSerdeInfo().hasSerializationLib()?
-          proto.getSerdeInfo().getSerializationLib():null);
-      serde.setParameters(buildParameters(proto.getSerdeInfo().getParameters()));
-      sd.setSerdeInfo(serde);
-    }
-    sd.setBucketCols(new ArrayList<>(proto.getBucketColsList()));
-    List<Order> sortCols = new ArrayList<>();
-    for (HbaseMetastoreProto.StorageDescriptor.Order protoOrder : proto.getSortColsList()) {
-      sortCols.add(new Order(protoOrder.getColumnName(), protoOrder.getOrder()));
-    }
-    sd.setSortCols(sortCols);
-    if (proto.hasSkewedInfo()) {
-      SkewedInfo skewed = new SkewedInfo();
-      skewed
-          .setSkewedColNames(new ArrayList<>(proto.getSkewedInfo().getSkewedColNamesList()));
-      for (HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueList innerList :
-          proto.getSkewedInfo().getSkewedColValuesList()) {
-        skewed.addToSkewedColValues(new ArrayList<>(innerList.getSkewedColValueList()));
-      }
-      Map<List<String>, String> colMaps = new HashMap<>();
-      for (HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueLocationMap map :
-          proto.getSkewedInfo().getSkewedColValueLocationMapsList()) {
-        colMaps.put(new ArrayList<>(map.getKeyList()), map.getValue());
-      }
-      skewed.setSkewedColValueLocationMaps(colMaps);
-      sd.setSkewedInfo(skewed);
-    }
-    if (proto.hasStoredAsSubDirectories()) {
-      sd.setStoredAsSubDirectories(proto.getStoredAsSubDirectories());
-    }
-    return sd;
-  }
-
-  static List<String> getPartitionKeyTypes(List<FieldSchema> parts) {
-    com.google.common.base.Function<FieldSchema, String> fieldSchemaToType =
-        new com.google.common.base.Function<FieldSchema, String>() {
-      public String apply(FieldSchema fs) { return fs.getType(); }
-    };
-    return Lists.transform(parts, fieldSchemaToType);
-  }
-
-  static List<String> getPartitionNames(List<FieldSchema> parts) {
-    com.google.common.base.Function<FieldSchema, String> fieldSchemaToName =
-        new com.google.common.base.Function<FieldSchema, String>() {
-      public String apply(FieldSchema fs) { return fs.getName(); }
-    };
-    return Lists.transform(parts, fieldSchemaToName);
-  }
-
-  /**
-   * Serialize a partition
-   * @param part partition object
-   * @param sdHash hash that is being used as a key for the enclosed storage descriptor
-   * @return First element is the key, second is the serialized partition
-   */
-  static byte[][] serializePartition(Partition part, List<String> partTypes, byte[] sdHash) {
-    byte[][] result = new byte[2][];
-    result[0] = buildPartitionKey(part.getDbName(), part.getTableName(), partTypes, part.getValues());
-    HbaseMetastoreProto.Partition.Builder builder = HbaseMetastoreProto.Partition.newBuilder();
-    builder
-        .setCreateTime(part.getCreateTime())
-        .setLastAccessTime(part.getLastAccessTime());
-    if (part.getSd().getLocation() != null) builder.setLocation(part.getSd().getLocation());
-    if (part.getSd().getParameters() != null) {
-      builder.setSdParameters(buildParameters(part.getSd().getParameters()));
-    }
-    builder.setSdHash(ByteString.copyFrom(sdHash));
-    if (part.getParameters() != null) builder.setParameters(buildParameters(part.getParameters()));
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-  static byte[] buildPartitionKey(String dbName, String tableName, List<String> partTypes, List<String> partVals) {
-    return buildPartitionKey(dbName, tableName, partTypes, partVals, false);
-  }
-
-  static byte[] buildPartitionKey(String dbName, String tableName, List<String> partTypes, List<String> partVals, boolean endPrefix) {
-    Object[] components = new Object[partVals.size()];
-    for (int i=0;i<partVals.size();i++) {
-      TypeInfo expectedType =
-          TypeInfoUtils.getTypeInfoFromTypeString(partTypes.get(i));
-      ObjectInspector outputOI =
-          TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(expectedType);
-      Converter converter = ObjectInspectorConverters.getConverter(
-          PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
-      components[i] = converter.convert(partVals.get(i));
-    }
-
-    return buildSerializedPartitionKey(dbName, tableName, partTypes, components, endPrefix);
-  }
-
-  static byte[] buildSerializedPartitionKey(String dbName, String tableName, List<String> partTypes, Object[] components, boolean endPrefix) {
-    ObjectInspector javaStringOI =
-        PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveCategory.STRING);
-    Object[] data = new Object[components.length+2];
-    List<ObjectInspector> fois = new ArrayList<ObjectInspector>(components.length+2);
-    boolean[] endPrefixes = new boolean[components.length+2];
-
-    data[0] = dbName;
-    fois.add(javaStringOI);
-    endPrefixes[0] = false;
-    data[1] = tableName;
-    fois.add(javaStringOI);
-    endPrefixes[1] = false;
-
-    for (int i = 0; i < components.length; i++) {
-      data[i+2] = components[i];
-      TypeInfo expectedType =
-          TypeInfoUtils.getTypeInfoFromTypeString(partTypes.get(i));
-      ObjectInspector outputOI =
-          TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(expectedType);
-      fois.add(outputOI);
-    }
-    Output output = new Output();
-    try {
-      BinarySortableSerDeWithEndPrefix.serializeStruct(output, data, fois, endPrefix);
-    } catch (SerDeException e) {
-      throw new RuntimeException("Cannot serialize partition " + StringUtils.join(components, ","));
-    }
-    return Arrays.copyOf(output.getData(), output.getLength());
-  }
-
-  static class StorageDescriptorParts {
-    byte[] sdHash;
-    String location;
-    Map<String, String> parameters;
-    Partition containingPartition;
-    Table containingTable;
-    Index containingIndex;
-  }
-
-  static void assembleStorageDescriptor(StorageDescriptor sd, StorageDescriptorParts parts) {
-    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
-    ssd.setLocation(parts.location);
-    ssd.setParameters(parts.parameters);
-    ssd.setShared(sd);
-    if (parts.containingPartition != null) {
-      parts.containingPartition.setSd(ssd);
-    } else if (parts.containingTable != null) {
-      parts.containingTable.setSd(ssd);
-    } else if (parts.containingIndex != null) {
-      parts.containingIndex.setSd(ssd);
-    }
-    else {
-      throw new RuntimeException("Need either a partition or a table");
-    }
-  }
-
-  /**
-   * Deserialize a partition key when you know nothing about it.  That is, you do not know what
-   * dbname, tablename it came from.
-   * @param key the key fetched from HBase
-   * @param callback A reference to the calling HBaseReadWrite object.  This has to be done as a
-   *                 callback because we have to first deserialize the database name and table
-   *                 name, and then fetch the table information, and then we will know how to
-   *                 desierliaze the rest of the key.
-   * @return a list that includes the dbname, tablename, and partition values
-   * @throws IOException
-   */
-  static List<String> deserializePartitionKey(byte[] key, HBaseReadWrite callback)
-      throws IOException {
-    List<String> keyParts =
-        desierliazeDbNameTableNameFromPartitionKey(key, callback.getConf());
-    Table table = callback.getTable(keyParts.get(0), keyParts.get(1));
-    keyParts.addAll(deserializePartitionKey(table.getPartitionKeys(), key, callback.getConf()));
-    return keyParts;
-  }
-
-   /**
-   * Deserialize a partition.  This version should be used when the partition key is not already
-   * known and the database and table name are not known either (eg a full scan).  Because the
-    * dbname and tablename (and thus the partition columns) are not known a priori this version
-    * has to go fetch the table after it figures out which table.  If you already have the table
-    * object you should use
-    * {@link #deserializePartition(String,String,List,byte[],byte[],Configuration)}
-   * @param key the key fetched from HBase
-   * @param serialized the value fetched from HBase
-   * @param callback A reference to the calling HBaseReadWrite object.  This has to be done as a
-   *                 callback because we have to first deserialize the database name and table
-   *                 name, and then fetch the table information, and then we will know how to
-   *                 desierliaze the rest of the key.
-   * @return A struct that contains the partition plus parts of the storage descriptor
-   */
-  static StorageDescriptorParts deserializePartition(byte[] key, byte[] serialized,
-                                                     HBaseReadWrite callback)
-      throws IOException {
-    List<String> dbNameTableName =
-        desierliazeDbNameTableNameFromPartitionKey(key, callback.getConf());
-    Table table = callback.getTable(dbNameTableName.get(0), dbNameTableName.get(1));
-    List<String> keys = deserializePartitionKey(table.getPartitionKeys(), key, callback.getConf());
-    return deserializePartition(dbNameTableName.get(0), dbNameTableName.get(1), keys, serialized);
-  }
-
-  /**
-   * Deserialize a partition.  This version should be used when you know the dbname and tablename
-   * but not the partition values.
-   * @param dbName database this partition is in
-   * @param tableName table this partition is in
-   * @param partitions schemas for the partition columns of this table
-   * @param key key fetched from HBase
-   * @param serialized serialized version of the partition
-   * @param conf configuration file
-   * @return
-   * @throws InvalidProtocolBufferException
-   */
-  static StorageDescriptorParts deserializePartition(String dbName, String tableName,
-                                                     List<FieldSchema> partitions, byte[] key,
-                                                     byte[] serialized, Configuration conf)
-      throws InvalidProtocolBufferException {
-    List<String> keys = deserializePartitionKey(partitions, key, conf);
-    return deserializePartition(dbName, tableName, keys, serialized);
-  }
-
-  /**
-   * Deserialize a partition.  This version should be used when the partition key is
-   * known (eg a get).
-   * @param dbName database name
-   * @param tableName table name
-   * @param partVals partition values
-   * @param serialized the value fetched from HBase
-   * @return A struct that contains the partition plus parts of the storage descriptor
-   */
-  static StorageDescriptorParts deserializePartition(String dbName, String tableName,
-                                                     List<String> partVals, byte[] serialized)
-      throws InvalidProtocolBufferException {
-    HbaseMetastoreProto.Partition proto = HbaseMetastoreProto.Partition.parseFrom(serialized);
-    Partition part = new Partition();
-    StorageDescriptorParts sdParts = new StorageDescriptorParts();
-    sdParts.containingPartition = part;
-    part.setDbName(dbName);
-    part.setTableName(tableName);
-    part.setValues(partVals);
-    part.setCreateTime((int)proto.getCreateTime());
-    part.setLastAccessTime((int)proto.getLastAccessTime());
-    if (proto.hasLocation()) sdParts.location = proto.getLocation();
-    if (proto.hasSdParameters()) sdParts.parameters = buildParameters(proto.getSdParameters());
-    sdParts.sdHash = proto.getSdHash().toByteArray();
-    if (proto.hasParameters()) part.setParameters(buildParameters(proto.getParameters()));
-    return sdParts;
-  }
-
-  static String[] deserializeKey(byte[] key) {
-    String k = new String(key, ENCODING);
-    return k.split(KEY_SEPARATOR_STR);
-  }
-
-  private static List<String> desierliazeDbNameTableNameFromPartitionKey(byte[] key,
-                                                                         Configuration conf) {
-    StringBuilder names = new StringBuilder();
-    names.append("dbName,tableName,");
-    StringBuilder types = new StringBuilder();
-    types.append("string,string,");
-    BinarySortableSerDe serDe = new BinarySortableSerDe();
-    Properties props = new Properties();
-    props.setProperty(serdeConstants.LIST_COLUMNS, names.toString());
-    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types.toString());
-    try {
-      serDe.initialize(conf, props);
-      List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(key))).subList(0, 2);
-      List<String> keys = new ArrayList<>();
-      for (int i=0;i<deserializedkeys.size();i++) {
-        Object deserializedKey = deserializedkeys.get(i);
-        if (deserializedKey==null) {
-          throw new RuntimeException("Can't have a null dbname or tablename");
-        } else {
-          TypeInfo inputType = TypeInfoUtils.getTypeInfoFromTypeString("string");
-          ObjectInspector inputOI =
-              TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(inputType);
-          Converter converter = ObjectInspectorConverters.getConverter(inputOI,
-              PrimitiveObjectInspectorFactory.javaStringObjectInspector);
-          keys.add((String) converter.convert(deserializedKey));
-        }
-      }
-      return keys;
-    } catch (SerDeException e) {
-      throw new RuntimeException("Error when deserialize key", e);
-    }
-  }
-
-  // Deserialize a partition key and return _only_ the partition values.
-  private static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key,
-      Configuration conf) {
-    StringBuilder names = new StringBuilder();
-    names.append("dbName,tableName,");
-    StringBuilder types = new StringBuilder();
-    types.append("string,string,");
-    for (int i=0;i<partitions.size();i++) {
-      names.append(partitions.get(i).getName());
-      types.append(TypeInfoUtils.getTypeInfoFromTypeString(partitions.get(i).getType()));
-      if (i!=partitions.size()-1) {
-        names.append(",");
-        types.append(",");
-      }
-    }
-    BinarySortableSerDe serDe = new BinarySortableSerDe();
-    Properties props = new Properties();
-    props.setProperty(serdeConstants.LIST_COLUMNS, names.toString());
-    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types.toString());
-    try {
-      serDe.initialize(conf, props);
-      List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(key))).subList(2, partitions.size()+2);
-      List<String> partitionKeys = new ArrayList<String>();
-      for (int i=0;i<deserializedkeys.size();i++) {
-        Object deserializedKey = deserializedkeys.get(i);
-        if (deserializedKey==null) {
-          partitionKeys.add(HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULTPARTITIONNAME));
-        } else {
-          TypeInfo inputType =
-              TypeInfoUtils.getTypeInfoFromTypeString(partitions.get(i).getType());
-          ObjectInspector inputOI =
-              TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(inputType);
-          Converter converter = ObjectInspectorConverters.getConverter(inputOI,
-              PrimitiveObjectInspectorFactory.javaStringObjectInspector);
-          partitionKeys.add((String)converter.convert(deserializedKey));
-        }
-      }
-      return partitionKeys;
-    } catch (SerDeException e) {
-      throw new RuntimeException("Error when deserialize key", e);
-    }
-  }
-
-  /**
-   * Serialize a table
-   * @param table table object
-   * @param sdHash hash that is being used as a key for the enclosed storage descriptor
-   * @return First element is the key, second is the serialized table
-   */
-  static byte[][] serializeTable(Table table, byte[] sdHash) {
-    byte[][] result = new byte[2][];
-    result[0] = buildKey(HiveStringUtils.normalizeIdentifier(table.getDbName()),
-        HiveStringUtils.normalizeIdentifier(table.getTableName()));
-    HbaseMetastoreProto.Table.Builder builder = HbaseMetastoreProto.Table.newBuilder();
-    if (table.getOwner() != null) builder.setOwner(table.getOwner());
-    builder
-        .setCreateTime(table.getCreateTime())
-        .setLastAccessTime(table.getLastAccessTime())
-        .setRetention(table.getRetention());
-    if (table.getSd().getLocation() != null) builder.setLocation(table.getSd().getLocation());
-    if (table.getSd().getParameters() != null) {
-      builder.setSdParameters(buildParameters(table.getSd().getParameters()));
-    }
-    builder.setSdHash(ByteString.copyFrom(sdHash));
-    if (table.getPartitionKeys() != null) {
-      builder.addAllPartitionKeys(convertFieldSchemaListToProto(table.getPartitionKeys()));
-    }
-    if (table.getParameters() != null) {
-      builder.setParameters(buildParameters(table.getParameters()));
-    }
-    if (table.getViewOriginalText() != null) {
-      builder.setViewOriginalText(table.getViewOriginalText());
-    }
-    if (table.getViewExpandedText() != null) {
-      builder.setViewExpandedText(table.getViewExpandedText());
-    }
-    builder.setIsRewriteEnabled(table.isRewriteEnabled());
-    if (table.getTableType() != null) builder.setTableType(table.getTableType());
-    if (table.getPrivileges() != null) {
-      builder.setPrivileges(buildPrincipalPrivilegeSet(table.getPrivileges()));
-    }
-    // Set only if table is temporary
-    if (table.isTemporary()) {
-      builder.setIsTemporary(table.isTemporary());
-    }
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-  /**
-   * Deserialize a table.  This version should be used when the table key is not already
-   * known (eg a scan).
-   * @param key the key fetched from HBase
-   * @param serialized the value fetched from HBase
-   * @return A struct that contains the table plus parts of the storage descriptor
-   */
-  static StorageDescriptorParts deserializeTable(byte[] key, byte[] serialized)
-      throws InvalidProtocolBufferException {
-    String[] keys = deserializeKey(key);
-    return deserializeTable(keys[0], keys[1], serialized);
-  }
-
-  /**
-   * Deserialize a table.  This version should be used when the table key is
-   * known (eg a get).
-   * @param dbName database name
-   * @param tableName table name
-   * @param serialized the value fetched from HBase
-   * @return A struct that contains the partition plus parts of the storage descriptor
-   */
-  static StorageDescriptorParts deserializeTable(String dbName, String tableName,
-                                                 byte[] serialized)
-      throws InvalidProtocolBufferException {
-    HbaseMetastoreProto.Table proto = HbaseMetastoreProto.Table.parseFrom(serialized);
-    Table table = new Table();
-    StorageDescriptorParts sdParts = new StorageDescriptorParts();
-    sdParts.containingTable = table;
-    table.setDbName(dbName);
-    table.setTableName(tableName);
-    table.setOwner(proto.getOwner());
-    table.setCreateTime((int)proto.getCreateTime());
-    table.setLastAccessTime((int)proto.getLastAccessTime());
-    table.setRetention((int)proto.getRetention());
-    if (proto.hasLocation()) sdParts.location = proto.getLocation();
-    if (proto.hasSdParameters()) sdParts.parameters = buildParameters(proto.getSdParameters());
-    sdParts.sdHash = proto.getSdHash().toByteArray();
-    table.setPartitionKeys(convertFieldSchemaListFromProto(proto.getPartitionKeysList()));
-    table.setParameters(buildParameters(proto.getParameters()));
-    if (proto.hasViewOriginalText()) table.setViewOriginalText(proto.getViewOriginalText());
-    if (proto.hasViewExpandedText()) table.setViewExpandedText(proto.getViewExpandedText());
-    table.setRewriteEnabled(proto.getIsRewriteEnabled());
-    table.setTableType(proto.getTableType());
-    if (proto.hasPrivileges()) {
-      table.setPrivileges(buildPrincipalPrivilegeSet(proto.getPrivileges()));
-    }
-    if (proto.hasIsTemporary()) table.setTemporary(proto.getIsTemporary());
-    return sdParts;
-  }
-
-  /**
-   * Serialize an index
-   * @param index index object
-   * @param sdHash hash that is being used as a key for the enclosed storage descriptor
-   * @return First element is the key, second is the serialized index
-   */
-  static byte[][] serializeIndex(Index index, byte[] sdHash) {
-    byte[][] result = new byte[2][];
-    result[0] = buildKey(HiveStringUtils.normalizeIdentifier(index.getDbName()),
-        HiveStringUtils.normalizeIdentifier(index.getOrigTableName()),
-        HiveStringUtils.normalizeIdentifier(index.getIndexName()));
-    HbaseMetastoreProto.Index.Builder builder = HbaseMetastoreProto.Index.newBuilder();
-    builder.setDbName(index.getDbName());
-    builder.setOrigTableName(index.getOrigTableName());
-    if (index.getSd().getLocation() != null) builder.setLocation(index.getSd().getLocation());
-    if (index.getSd().getParameters() != null) {
-      builder.setSdParameters(buildParameters(index.getSd().getParameters()));
-    }
-    if (index.getIndexHandlerClass() != null) {
-      builder.setIndexHandlerClass(index.getIndexHandlerClass());
-    }
-    if (index.getIndexTableName() != null) {
-      builder.setIndexTableName(index.getIndexTableName());
-    }
-    builder
-        .setCreateTime(index.getCreateTime())
-        .setLastAccessTime(index.getLastAccessTime())
-        .setDeferredRebuild(index.isDeferredRebuild());
-    if (index.getParameters() != null) {
-      builder.setParameters(buildParameters(index.getParameters()));
-    }
-    if (sdHash != null) {
-      builder.setSdHash(ByteString.copyFrom(sdHash));
-    }
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-  /**
-   * Deserialize an index.  This version should be used when the index key is not already
-   * known (eg a scan).
-   * @param key the key fetched from HBase
-   * @param serialized the value fetched from HBase
-   * @return A struct that contains the index plus parts of the storage descriptor
-   */
-  static StorageDescriptorParts deserializeIndex(byte[] key, byte[] serialized)
-      throws InvalidProtocolBufferException {
-    String[] keys = deserializeKey(key);
-    return deserializeIndex(keys[0], keys[1], keys[2], serialized);
-  }
-
-  /**
-   * Deserialize an index.  This version should be used when the table key is
-   * known (eg a get).
-   * @param dbName database name
-   * @param origTableName original table name
-   * @param indexName index name
-   * @param serialized the value fetched from HBase
-   * @return A struct that contains the index plus parts of the storage descriptor
-   */
-  static StorageDescriptorParts deserializeIndex(String dbName, String origTableName,
-                                                 String indexName, byte[] serialized)
-      throws InvalidProtocolBufferException {
-    HbaseMetastoreProto.Index proto = HbaseMetastoreProto.Index.parseFrom(serialized);
-    Index index = new Index();
-    StorageDescriptorParts sdParts = new StorageDescriptorParts();
-    sdParts.containingIndex = index;
-    index.setDbName(dbName);
-    index.setIndexName(indexName);
-    index.setOrigTableName(origTableName);
-    if (proto.hasLocation()) sdParts.location = proto.getLocation();
-    if (proto.hasSdParameters()) sdParts.parameters = buildParameters(proto.getSdParameters());
-    if (proto.hasIndexHandlerClass()) {
-      index.setIndexHandlerClass(proto.getIndexHandlerClass());
-    }
-    if (proto.hasIndexTableName()) {
-      index.setIndexTableName(proto.getIndexTableName());
-    }
-    index.setCreateTime(proto.getCreateTime());
-    index.setLastAccessTime(proto.getLastAccessTime());
-    index.setDeferredRebuild(proto.getDeferredRebuild());
-    index.setParameters(buildParameters(proto.getParameters()));
-    if (proto.hasSdHash()) {
-      sdParts.sdHash = proto.getSdHash().toByteArray();
-    }
-    return sdParts;
-  }
-
-  static byte[] serializeBloomFilter(String dbName, String tableName, BloomFilter bloom) {
-    long[] bitSet = bloom.getBitSet();
-    List<Long> bits = new ArrayList<>(bitSet.length);
-    for (int i = 0; i < bitSet.length; i++) bits.add(bitSet[i]);
-    HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter protoBloom =
-        HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.newBuilder()
-        .setNumBits(bloom.getBitSize())
-        .setNumFuncs(bloom.getNumHashFunctions())
-        .addAllBits(bits)
-        .build();
-
-    HbaseMetastoreProto.AggrStatsBloomFilter proto =
-        HbaseMetastoreProto.AggrStatsBloomFilter.newBuilder()
-            .setDbName(ByteString.copyFrom(dbName.getBytes(ENCODING)))
-            .setTableName(ByteString.copyFrom(tableName.getBytes(ENCODING)))
-            .setBloomFilter(protoBloom)
-            .setAggregatedAt(System.currentTimeMillis())
-            .build();
-
-    return proto.toByteArray();
-  }
-
-  private static HbaseMetastoreProto.ColumnStats protoBufStatsForOneColumn(
-      ColumnStatistics partitionColumnStats, ColumnStatisticsObj colStats) throws IOException {
-    HbaseMetastoreProto.ColumnStats.Builder builder = HbaseMetastoreProto.ColumnStats.newBuilder();
-    if (partitionColumnStats != null) {
-      builder.setLastAnalyzed(partitionColumnStats.getStatsDesc().getLastAnalyzed());
-    }
-    assert colStats.getColType() != null;
-    builder.setColumnType(colStats.getColType());
-    assert colStats.getColName() != null;
-    builder.setColumnName(colStats.getColName());
-
-    ColumnStatisticsData colData = colStats.getStatsData();
-    switch (colData.getSetField()) {
-    case BOOLEAN_STATS:
-      BooleanColumnStatsData boolData = colData.getBooleanStats();
-      builder.setNumNulls(boolData.getNumNulls());
-      builder.setBoolStats(HbaseMetastoreProto.ColumnStats.BooleanStats.newBuilder()
-          .setNumTrues(boolData.getNumTrues()).setNumFalses(boolData.getNumFalses()).build());
-      break;
-
-    case LONG_STATS:
-      LongColumnStatsData longData = colData.getLongStats();
-      builder.setNumNulls(longData.getNumNulls());
-      builder.setNumDistinctValues(longData.getNumDVs());
-      if (longData.isSetBitVectors()) {
-        builder.setBitVectors(longData.getBitVectors());
-      }
-      builder.setLongStats(HbaseMetastoreProto.ColumnStats.LongStats.newBuilder()
-          .setLowValue(longData.getLowValue()).setHighValue(longData.getHighValue()).build());
-      break;
-
-    case DOUBLE_STATS:
-      DoubleColumnStatsData doubleData = colData.getDoubleStats();
-      builder.setNumNulls(doubleData.getNumNulls());
-      builder.setNumDistinctValues(doubleData.getNumDVs());
-      if (doubleData.isSetBitVectors()) {
-        builder.setBitVectors(doubleData.getBitVectors());
-      }
-      builder.setDoubleStats(HbaseMetastoreProto.ColumnStats.DoubleStats.newBuilder()
-          .setLowValue(doubleData.getLowValue()).setHighValue(doubleData.getHighValue()).build());
-      break;
-
-    case STRING_STATS:
-      StringColumnStatsData stringData = colData.getStringStats();
-      builder.setNumNulls(stringData.getNumNulls());
-      builder.setNumDistinctValues(stringData.getNumDVs());
-      if (stringData.isSetBitVectors()) {
-        builder.setBitVectors(stringData.getBitVectors());
-      }
-      builder.setStringStats(HbaseMetastoreProto.ColumnStats.StringStats.newBuilder()
-          .setMaxColLength(stringData.getMaxColLen()).setAvgColLength(stringData.getAvgColLen())
-          .build());
-      break;
-
-    case BINARY_STATS:
-      BinaryColumnStatsData binaryData = colData.getBinaryStats();
-      builder.setNumNulls(binaryData.getNumNulls());
-      builder.setBinaryStats(HbaseMetastoreProto.ColumnStats.StringStats.newBuilder()
-          .setMaxColLength(binaryData.getMaxColLen()).setAvgColLength(binaryData.getAvgColLen())
-          .build());
-      break;
-
-    case DECIMAL_STATS:
-      DecimalColumnStatsData decimalData = colData.getDecimalStats();
-      builder.setNumNulls(decimalData.getNumNulls());
-      builder.setNumDistinctValues(decimalData.getNumDVs());
-      if (decimalData.isSetBitVectors()) {
-        builder.setBitVectors(decimalData.getBitVectors());
-      }
-      if (decimalData.getLowValue() != null && decimalData.getHighValue() != null) {
-        builder.setDecimalStats(
-            HbaseMetastoreProto.ColumnStats.DecimalStats
-                .newBuilder()
-                .setLowValue(
-                    HbaseMetastoreProto.ColumnStats.DecimalStats.Decimal.newBuilder()
-                        .setUnscaled(ByteString.copyFrom(decimalData.getLowValue().getUnscaled()))
-                        .setScale(decimalData.getLowValue().getScale()).build())
-                .setHighValue(
-                    HbaseMetastoreProto.ColumnStats.DecimalStats.Decimal.newBuilder()
-                        .setUnscaled(ByteString.copyFrom(decimalData.getHighValue().getUnscaled()))
-                        .setScale(decimalData.getHighValue().getScale()).build())).build();
-      } else {
-        builder.setDecimalStats(HbaseMetastoreProto.ColumnStats.DecimalStats.newBuilder().clear()
-            .build());
-      }
-      break;
-
-    default:
-      throw new RuntimeException("Woh, bad.  Unknown stats type!");
-    }
-    return builder.build();
-  }
-
-  static byte[] serializeStatsForOneColumn(ColumnStatistics partitionColumnStats,
-                                           ColumnStatisticsObj colStats) throws IOException {
-    return protoBufStatsForOneColumn(partitionColumnStats, colStats).toByteArray();
-  }
-
-  static ColumnStatisticsObj deserializeStatsForOneColumn(ColumnStatistics partitionColumnStats,
-      byte[] bytes) throws IOException {
-    HbaseMetastoreProto.ColumnStats proto = HbaseMetastoreProto.ColumnStats.parseFrom(bytes);
-    return statsForOneColumnFromProtoBuf(partitionColumnStats, proto);
-  }
-
-  private static ColumnStatisticsObj
-  statsForOneColumnFromProtoBuf(ColumnStatistics partitionColumnStats,
-                                HbaseMetastoreProto.ColumnStats proto) throws IOException {
-    ColumnStatisticsObj colStats = new ColumnStatisticsObj();
-    long lastAnalyzed = proto.getLastAnalyzed();
-    if (partitionColumnStats != null) {
-      partitionColumnStats.getStatsDesc().setLastAnalyzed(
-          Math.max(lastAnalyzed, partitionColumnStats.getStatsDesc().getLastAnalyzed()));
-    }
-    colStats.setColType(proto.getColumnType());
-    colStats.setColName(proto.getColumnName());
-
-    ColumnStatisticsData colData = new ColumnStatisticsData();
-    if (proto.hasBoolStats()) {
-      BooleanColumnStatsData boolData = new BooleanColumnStatsData();
-      boolData.setNumTrues(proto.getBoolStats().getNumTrues());
-      boolData.setNumFalses(proto.getBoolStats().getNumFalses());
-      boolData.setNumNulls(proto.getNumNulls());
-      colData.setBooleanStats(boolData);
-    } else if (proto.hasLongStats()) {
-      LongColumnStatsData longData = new LongColumnStatsData();
-      if (proto.getLongStats().hasLowValue()) {
-        longData.setLowValue(proto.getLongStats().getLowValue());
-      }
-      if (proto.getLongStats().hasHighValue()) {
-        longData.setHighValue(proto.getLongStats().getHighValue());
-      }
-      longData.setNumNulls(proto.getNumNulls());
-      longData.setNumDVs(proto.getNumDistinctValues());
-      longData.setBitVectors(proto.getBitVectors());
-      colData.setLongStats(longData);
-    } else if (proto.hasDoubleStats()) {
-      DoubleColumnStatsData doubleData = new DoubleColumnStatsData();
-      if (proto.getDoubleStats().hasLowValue()) {
-        doubleData.setLowValue(proto.getDoubleStats().getLowValue());
-      }
-      if (proto.getDoubleStats().hasHighValue()) {
-        doubleData.setHighValue(proto.getDoubleStats().getHighValue());
-      }
-      doubleData.setNumNulls(proto.getNumNulls());
-      doubleData.setNumDVs(proto.getNumDistinctValues());
-      doubleData.setBitVectors(proto.getBitVectors());
-      colData.setDoubleStats(doubleData);
-    } else if (proto.hasStringStats()) {
-      StringColumnStatsData stringData = new StringColumnStatsData();
-      stringData.setMaxColLen(proto.getStringStats().getMaxColLength());
-      stringData.setAvgColLen(proto.getStringStats().getAvgColLength());
-      stringData.setNumNulls(proto.getNumNulls());
-      stringData.setNumDVs(proto.getNumDistinctValues());
-      stringData.setBitVectors(proto.getBitVectors());
-      colData.setStringStats(stringData);
-    } else if (proto.hasBinaryStats()) {
-      BinaryColumnStatsData binaryData = new BinaryColumnStatsData();
-      binaryData.setMaxColLen(proto.getBinaryStats().getMaxColLength());
-      binaryData.setAvgColLen(proto.getBinaryStats().getAvgColLength());
-      binaryData.setNumNulls(proto.getNumNulls());
-      colData.setBinaryStats(binaryData);
-    } else if (proto.hasDecimalStats()) {
-      DecimalColumnStatsData decimalData = new DecimalColumnStatsData();
-      if (proto.getDecimalStats().hasHighValue()) {
-        Decimal hiVal = new Decimal();
-        hiVal.setUnscaled(proto.getDecimalStats().getHighValue().getUnscaled().toByteArray());
-        hiVal.setScale((short) proto.getDecimalStats().getHighValue().getScale());
-        decimalData.setHighValue(hiVal);
-      }
-      if (proto.getDecimalStats().hasLowValue()) {
-        Decimal loVal = new Decimal();
-        loVal.setUnscaled(proto.getDecimalStats().getLowValue().getUnscaled().toByteArray());
-        loVal.setScale((short) proto.getDecimalStats().getLowValue().getScale());
-        decimalData.setLowValue(loVal);
-      }
-      decimalData.setNumNulls(proto.getNumNulls());
-      decimalData.setNumDVs(proto.getNumDistinctValues());
-      decimalData.setBitVectors(proto.getBitVectors());
-      colData.setDecimalStats(decimalData);
-    } else {
-      throw new RuntimeException("Woh, bad.  Unknown stats type!");
-    }
-    colStats.setStatsData(colData);
-    return colStats;
-  }
-
-  static byte[] serializeAggrStats(AggrStats aggrStats) throws IOException {
-    List<HbaseMetastoreProto.ColumnStats> protoColStats =
-        new ArrayList<>(aggrStats.getColStatsSize());
-    for (ColumnStatisticsObj cso : aggrStats.getColStats()) {
-      protoColStats.add(protoBufStatsForOneColumn(null, cso));
-    }
-    return HbaseMetastoreProto.AggrStats.newBuilder()
-        .setPartsFound(aggrStats.getPartsFound())
-        .addAllColStats(protoColStats)
-        .build()
-        .toByteArray();
-  }
-
-  static AggrStats deserializeAggrStats(byte[] serialized) throws IOException {
-    HbaseMetastoreProto.AggrStats protoAggrStats =
-        HbaseMetastoreProto.AggrStats.parseFrom(serialized);
-    AggrStats aggrStats = new AggrStats();
-    aggrStats.setPartsFound(protoAggrStats.getPartsFound());
-    for (HbaseMetastoreProto.ColumnStats protoCS : protoAggrStats.getColStatsList()) {
-      aggrStats.addToColStats(statsForOneColumnFromProtoBuf(null, protoCS));
-    }
-    return aggrStats;
-  }
-
-  /**
-   * Serialize a delegation token
-   * @param tokenIdentifier
-   * @param delegationToken
-   * @return two byte arrays, first contains the key, the second the serialized value.
-   */
-  static byte[][] serializeDelegationToken(String tokenIdentifier, String delegationToken) {
-    byte[][] result = new byte[2][];
-    result[0] = buildKey(tokenIdentifier);
-    result[1] = HbaseMetastoreProto.DelegationToken.newBuilder()
-        .setTokenStr(delegationToken)
-        .build()
-        .toByteArray();
-    return result;
-  }
-
-  /**
-   * Deserialize a delegation token.
-   * @param value value fetched from hbase
-   * @return A delegation token.
-   * @throws InvalidProtocolBufferException
-   */
-  static String deserializeDelegationToken(byte[] value) throws InvalidProtocolBufferException {
-    HbaseMetastoreProto.DelegationToken protoToken =
-        HbaseMetastoreProto.DelegationToken.parseFrom(value);
-    return protoToken.getTokenStr();
-  }
-
-  /**
-   * Serialize a master key
-   * @param seqNo
-   * @param key
-   * @return two byte arrays, first contains the key, the second the serialized value.
-   */
-  static byte[][] serializeMasterKey(Integer seqNo, String key) {
-    byte[][] result = new byte[2][];
-    result[0] = buildKey(seqNo.toString());
-    result[1] = HbaseMetastoreProto.MasterKey.newBuilder()
-        .setMasterKey(key)
-        .build()
-        .toByteArray();
-    return result;
-  }
-
-  /**
-   * Deserialize a master key.
-   * @param value value fetched from hbase
-   * @return A master key
-   * @throws InvalidProtocolBufferException
-   */
-  static String deserializeMasterKey(byte[] value) throws InvalidProtocolBufferException {
-    HbaseMetastoreProto.MasterKey protoKey = HbaseMetastoreProto.MasterKey.parseFrom(value);
-    return protoKey.getMasterKey();
-  }
-
-  /**
-   * Serialize the primary key for a table.
-   * @param pk Primary key columns.  It is expected that all of these match to one pk, since
-   *           anything else is meaningless.
-   * @return two byte arrays, first containts the hbase key, the second the serialized value.
-   */
-  static byte[][] serializePrimaryKey(List<SQLPrimaryKey> pk) {
-    // First, figure out the dbName and tableName.  We expect this to match for all list entries.
-    byte[][] result = new byte[2][];
-    String dbName = pk.get(0).getTable_db();
-    String tableName = pk.get(0).getTable_name();
-    result[0] = buildKey(HiveStringUtils.normalizeIdentifier(dbName),
-        HiveStringUtils.normalizeIdentifier(tableName));
-
-    HbaseMetastoreProto.PrimaryKey.Builder builder = HbaseMetastoreProto.PrimaryKey.newBuilder();
-    // Encode the primary key, if present
-    builder.setPkName(pk.get(0).getPk_name());
-    builder.setEnableConstraint(pk.get(0).isEnable_cstr());
-    builder.setValidateConstraint(pk.get(0).isValidate_cstr());
-    builder.setRelyConstraint(pk.get(0).isRely_cstr());
-
-    for (SQLPrimaryKey pkcol : pk) {
-      HbaseMetastoreProto.PrimaryKey.PrimaryKeyColumn.Builder pkColBuilder =
-          HbaseMetastoreProto.PrimaryKey.PrimaryKeyColumn.newBuilder();
-      pkColBuilder.setColumnName(pkcol.getColumn_name());
-      pkColBuilder.setKeySeq(pkcol.getKey_seq());
-      builder.addCols(pkColBuilder);
-    }
-
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-  /**
-   * Serialize the foreign key(s) for a table.
-   * @param fks Foreign key columns.  These may belong to multiple foreign keys.
-   * @return two byte arrays, first containts the key, the second the serialized value.
-   */
-  static byte[][] serializeForeignKeys(List<SQLForeignKey> fks) {
-    // First, figure out the dbName and tableName.  We expect this to match for all list entries.
-    byte[][] result = new byte[2][];
-    String dbName = fks.get(0).getFktable_db();
-    String tableName = fks.get(0).getFktable_name();
-    result[0] = buildKey(HiveStringUtils.normalizeIdentifier(dbName),
-        HiveStringUtils.normalizeIdentifier(tableName));
-
-    HbaseMetastoreProto.ForeignKeys.Builder builder = HbaseMetastoreProto.ForeignKeys.newBuilder();
-
-    // Encode any foreign keys we find.  This can be complex because there may be more than
-    // one foreign key in here, so we need to detect that.
-    Map<String, HbaseMetastoreProto.ForeignKeys.ForeignKey.Builder> fkBuilders = new HashMap<>();
-
-    for (SQLForeignKey fkcol : fks) {
-      HbaseMetastoreProto.ForeignKeys.ForeignKey.Builder fkBuilder =
-          fkBuilders.get(fkcol.getFk_name());
-      if (fkBuilder == null) {
-        // We haven't seen this key before, so add it
-        fkBuilder = HbaseMetastoreProto.ForeignKeys.ForeignKey.newBuilder();
-        fkBuilder.setFkName(fkcol.getFk_name());
-        fkBuilder.setReferencedDbName(fkcol.getPktable_db());
-        assert dbName.equals(fkcol.getFktable_db()) : "You switched databases on me!";
-        fkBuilder.setReferencedTableName(fkcol.getPktable_name());
-        assert tableName.equals(fkcol.getFktable_name()) : "You switched tables on me!";
-        fkBuilder.setReferencedPkName(fkcol.getPk_name());
-        fkBuilder.setUpdateRule(fkcol.getUpdate_rule());
-        fkBuilder.setDeleteRule(fkcol.getDelete_rule());
-        fkBuilder.setEnableConstraint(fkcol.isEnable_cstr());
-        fkBuilder.setValidateConstraint(fkcol.isValidate_cstr());
-        fkBuilder.setRelyConstraint(fkcol.isRely_cstr());
-        fkBuilders.put(fkcol.getFk_name(), fkBuilder);
-      }
-      HbaseMetastoreProto.ForeignKeys.ForeignKey.ForeignKeyColumn.Builder fkColBuilder =
-          HbaseMetastoreProto.ForeignKeys.ForeignKey.ForeignKeyColumn.newBuilder();
-      fkColBuilder.setColumnName(fkcol.getFkcolumn_name());
-      fkColBuilder.setReferencedColumnName(fkcol.getPkcolumn_name());
-      fkColBuilder.setKeySeq(fkcol.getKey_seq());
-      fkBuilder.addCols(fkColBuilder);
-    }
-    for (HbaseMetastoreProto.ForeignKeys.ForeignKey.Builder fkBuilder : fkBuilders.values()) {
-      builder.addFks(fkBuilder);
-    }
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-  /**
-   * Serialize the unique constraint(s) for a table.
-   * @param uks Unique constraint columns.  These may belong to multiple unique constraints.
-   * @return two byte arrays, first contains the key, the second the serialized value.
-   */
-  static byte[][] serializeUniqueConstraints(List<SQLUniqueConstraint> uks) {
-    // First, figure out the dbName and tableName.  We expect this to match for all list entries.
-    byte[][] result = new byte[2][];
-    String dbName = uks.get(0).getTable_db();
-    String tableName = uks.get(0).getTable_name();
-    result[0] = buildKey(HiveStringUtils.normalizeIdentifier(dbName),
-        HiveStringUtils.normalizeIdentifier(tableName));
-
-    HbaseMetastoreProto.UniqueConstraints.Builder builder =
-        HbaseMetastoreProto.UniqueConstraints.newBuilder();
-
-    // Encode any foreign keys we find.  This can be complex because there may be more than
-    // one foreign key in here, so we need to detect that.
-    Map<String, HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.Builder> ukBuilders = new HashMap<>();
-
-    for (SQLUniqueConstraint ukcol : uks) {
-      HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.Builder ukBuilder =
-          ukBuilders.get(ukcol.getUk_name());
-      if (ukBuilder == null) {
-        // We haven't seen this key before, so add it
-        ukBuilder = HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.newBuilder();
-        ukBuilder.setUkName(ukcol.getUk_name());
-        ukBuilder.setEnableConstraint(ukcol.isEnable_cstr());
-        ukBuilder.setValidateConstraint(ukcol.isValidate_cstr());
-        ukBuilder.setRelyConstraint(ukcol.isRely_cstr());
-        ukBuilders.put(ukcol.getUk_name(), ukBuilder);
-      }
-      assert dbName.equals(ukcol.getTable_db()) : "You switched databases on me!";
-      assert tableName.equals(ukcol.getTable_name()) : "You switched tables on me!";
-      HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.UniqueConstraintColumn.Builder ukColBuilder =
-          HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.UniqueConstraintColumn.newBuilder();
-      ukColBuilder.setColumnName(ukcol.getColumn_name());
-      ukColBuilder.setKeySeq(ukcol.getKey_seq());
-      ukBuilder.addCols(ukColBuilder);
-    }
-    for (HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.Builder ukBuilder : ukBuilders.values()) {
-      builder.addUks(ukBuilder);
-    }
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-  /**
-   * Serialize the not null constraint(s) for a table.
-   * @param nns Not null constraint columns.  These may belong to multiple constraints.
-   * @return two byte arrays, first contains the constraint, the second the serialized value.
-   */
-  static byte[][] serializeNotNullConstraints(List<SQLNotNullConstraint> nns) {
-    // First, figure out the dbName and tableName.  We expect this to match for all list entries.
-    byte[][] result = new byte[2][];
-    String dbName = nns.get(0).getTable_db();
-    String tableName = nns.get(0).getTable_name();
-    result[0] = buildKey(HiveStringUtils.normalizeIdentifier(dbName),
-        HiveStringUtils.normalizeIdentifier(tableName));
-
-    HbaseMetastoreProto.NotNullConstraints.Builder builder =
-        HbaseMetastoreProto.NotNullConstraints.newBuilder();
-
-    // Encode any foreign keys we find.  This can be complex because there may be more than
-    // one foreign key in here, so we need to detect that.
-    Map<String, HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.Builder> nnBuilders = new HashMap<>();
-
-    for (SQLNotNullConstraint nncol : nns) {
-      HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.Builder nnBuilder =
-          nnBuilders.get(nncol.getNn_name());
-      if (nnBuilder == null) {
-        // We haven't seen this key before, so add it
-        nnBuilder = HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.newBuilder();
-        nnBuilder.setNnName(nncol.getNn_name());
-        nnBuilder.setEnableConstraint(nncol.isEnable_cstr());
-        nnBuilder.setValidateConstraint(nncol.isValidate_cstr());
-        nnBuilder.setRelyConstraint(nncol.isRely_cstr());
-        nnBuilders.put(nncol.getNn_name(), nnBuilder);
-      }
-      assert dbName.equals(nncol.getTable_db()) : "You switched databases on me!";
-      assert tableName.equals(nncol.getTable_name()) : "You switched tables on me!";
-      HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.NotNullConstraintColumn.Builder nnColBuilder =
-          HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.NotNullConstraintColumn.newBuilder();
-      nnColBuilder.setColumnName(nncol.getColumn_name());
-      nnBuilder.addCols(nnColBuilder);
-    }
-    for (HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.Builder nnBuilder : nnBuilders.values()) {
-      builder.addNns(nnBuilder);
-    }
-    result[1] = builder.build().toByteArray();
-    return result;
-  }
-
-  static List<SQLPrimaryKey> deserializePrimaryKey(String dbName, String tableName, byte[] value)
-      throws InvalidProtocolBufferException {
-    HbaseMetastoreProto.PrimaryKey proto = HbaseMetastoreProto.PrimaryKey.parseFrom(value);
-    List<SQLPrimaryKey> result = new ArrayList<>();
-    for (HbaseMetastoreProto.PrimaryKey.PrimaryKeyColumn protoPkCol : proto.getColsList()) {
-      result.add(new SQLPrimaryKey(dbName, tableName, protoPkCol.getColumnName(),
-          protoPkCol.getKeySeq(), proto.getPkName(), proto.getEnableConstraint(),
-          proto.getValidateConstraint(), proto.getRelyConstraint()));
-    }
-
-    return result;
-  }
-
-  static List<SQLUniqueConstraint> deserializeUniqueConstraint(String dbName, String tableName, byte[] value)
-      throws InvalidProtocolBufferException {
-    List<SQLUniqueConstraint> result = new ArrayList<>();
-    HbaseMetastoreProto.UniqueConstraints protoConstraints =
-        HbaseMetastoreProto.UniqueConstraints.parseFrom(value);
-
-    for (HbaseMetastoreProto.UniqueConstraints.UniqueConstraint proto : protoConstraints.getUksList()) {
-      for (HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.UniqueConstraintColumn protoUkCol :
-          proto.getColsList()) {
-        result.add(new SQLUniqueConstraint(dbName, tableName, protoUkCol.getColumnName(),
-            protoUkCol.getKeySeq(),
-            proto.getUkName(), proto.getEnableConstraint(),
-            proto.getValidateConstraint(), proto.getRelyConstraint()));
-      }
-    }
-    return result;
-  }
-
-  static List<SQLNotNullConstraint> deserializeNotNullConstraint(String dbName, String tableName, byte[] value)
-      throws InvalidProtocolBufferException {
-    List<SQLNotNullConstraint> result = new ArrayList<>();
-    HbaseMetastoreProto.NotNullConstraints protoConstraints =
-        HbaseMetastoreProto.NotNullConstraints.parseFrom(value);
-
-    for (HbaseMetastoreProto.NotNullConstraints.NotNullConstraint proto : protoConstraints.getNnsList()) {
-      for (HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.NotNullConstraintColumn protoNnCol :
-          proto.getColsList()) {
-        result.add(new SQLNotNullConstraint(dbName, tableName, protoNnCol.getColumnName(),
-            proto.getNnName(), proto.getEnableConstraint(),
-            proto.getValidateConstraint(), proto.getRelyConstraint()));
-      }
-    }
-    return result;
-  }
-
-  static List<SQLForeignKey> deserializeForeignKeys(String dbName, String tableName, byte[] value)
-      throws InvalidProtocolBufferException {
-    List<SQLForeignKey> result = new ArrayList<>();
-    HbaseMetastoreProto.ForeignKeys protoConstraints =
-        HbaseMetastoreProto.ForeignKeys.parseFrom(value);
-
-    for (HbaseMetastoreProto.ForeignKeys.ForeignKey protoFk : protoConstraints.getFksList()) {
-      for (HbaseMetastoreProto.ForeignKeys.ForeignKey.ForeignKeyColumn protoFkCol :
-          protoFk.getColsList()) {
-        result.add(new SQLForeignKey(protoFk.getReferencedDbName(), protoFk.getReferencedTableName(),
-            protoFkCol.getReferencedColumnName(), dbName, tableName, protoFkCol.getColumnName(),
-            protoFkCol.getKeySeq(), protoFk.getUpdateRule(), protoFk.getDeleteRule(),
-            protoFk.getFkName(), protoFk.getReferencedPkName(), protoFk.getEnableConstraint(),
-            protoFk.getValidateConstraint(), protoFk.getRelyConstraint()));
-      }
-    }
-    return result;
-  }
-
-  /**
-   * @param keyStart byte array representing the start prefix
-   * @return byte array corresponding to the next possible prefix
-   */
-  static byte[] getEndPrefix(byte[] keyStart) {
-    if (keyStart == null) {
-      return null;
-    }
-    // Since this is a prefix and not full key, the usual hbase technique of
-    // appending 0 byte does not work. Instead of that, increment the last byte.
-    byte[] keyEnd = Arrays.copyOf(keyStart, keyStart.length);
-    keyEnd[keyEnd.length - 1]++;
-    return keyEnd;
-  }
-
-  static byte[] makeLongKey(long v) {
-    byte[] b = new byte[8];
-    b[0] = (byte)(v >>> 56);
-    b[1] = (byte)(v >>> 48);
-    b[2] = (byte)(v >>> 40);
-    b[3] = (byte)(v >>> 32);
-    b[4] = (byte)(v >>> 24);
-    b[5] = (byte)(v >>> 16);
-    b[6] = (byte)(v >>>  8);
-    b[7] = (byte)(v >>>  0);
-    return b;
-  }
-
-  public static double getDoubleValue(Decimal decimal) {
-    return new BigDecimal(new BigInteger(decimal.getUnscaled()), decimal.getScale()).doubleValue();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/ObjectCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/ObjectCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/ObjectCache.java
deleted file mode 100644
index 5bbed5d..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/ObjectCache.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A generic class for caching objects obtained from HBase.  Currently a set of
- * convenience methods around a {@link java.util.HashMap} with a max size but built
- * as a separate class in case we want to switch out the implementation to something more
- * efficient.  The cache has a max size; when this is exceeded any additional entries are dropped
- * on the floor.
- *
- * This cache is local to a particular thread and thus is not synchronized.  It is intended to be
- * flushed before a query begins to make sure it doesn't carry old versions of objects between
- * queries (that is, an object may have changed between two queries, we want to get the newest
- * version).
- */
-class ObjectCache<K, V> {
-  private Map<K, V> cache;
-  private final int maxSize;
-  private Counter hits;
-  private Counter misses;
-  private Counter overflows;
-
-  /**
-   *
-   * @param max maximum number of objects to store in the cache.  When max is reached, eviction
-   *            policy is MRU.
-   * @param hits counter to increment when we find an element in the cache
-   * @param misses counter to increment when we do not find an element in the cache
-   * @param overflows counter to increment when we do not have room for an element in the cache
-   */
-  ObjectCache(int max, Counter hits, Counter misses, Counter overflows) {
-    maxSize = max;
-    cache = new HashMap<K, V>();
-    this.hits = hits;
-    this.misses = misses;
-    this.overflows = overflows;
-  }
-
-  void put(K key, V value) {
-    if (cache.size() < maxSize) {
-      cache.put(key, value);
-    } else {
-      overflows.incr();
-    }
-  }
-
-  V get(K key) {
-    V val = cache.get(key);
-    if (val == null) misses.incr();
-    else hits.incr();
-    return val;
-  }
-
-  void remove(K key) {
-    cache.remove(key);
-  }
-
-  void flush() {
-    cache.clear();
-  }
-}


[14/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
deleted file mode 100644
index c29e46a..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Integration tests with HBase Mini-cluster for HBaseStore
- */
-public class TestStorageDescriptorSharing extends HBaseIntegrationTests {
-
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreIntegration.class.getName());
-
-  private MessageDigest md;
-
-  @BeforeClass
-  public static void startup() throws Exception {
-    HBaseIntegrationTests.startMiniCluster();
-  }
-
-  @AfterClass
-  public static void shutdown() throws Exception {
-    HBaseIntegrationTests.shutdownMiniCluster();
-  }
-
-  @Before
-  public void setup() throws IOException {
-    setupConnection();
-    setupHBaseStore();
-    try {
-      md = MessageDigest.getInstance("MD5");
-    } catch (NoSuchAlgorithmException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Test
-  public void createManyPartitions() throws Exception {
-    String dbName = "default";
-    String tableName = "manyParts";
-    int startTime = (int)(System.currentTimeMillis() / 1000);
-    List<FieldSchema> cols = new ArrayList<FieldSchema>();
-    cols.add(new FieldSchema("col1", "int", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, emptyParameters);
-    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
-    partCols.add(new FieldSchema("pc", "string", ""));
-    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
-    for (String val : partVals) {
-      List<String> vals = new ArrayList<String>();
-      vals.add(val);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/pc=" + val);
-      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
-          emptyParameters);
-      store.addPartition(part);
-
-      Partition p = store.getPartition(dbName, tableName, vals);
-      Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
-    }
-
-    Assert.assertEquals(1, HBaseReadWrite.getInstance().countStorageDescriptor());
-
-    String tableName2 = "differentTable";
-    sd = new StorageDescriptor(cols, "file:/tmp", "input2", "output", false, 0,
-        serde, null, null, emptyParameters);
-    table = new Table(tableName2, "default", "me", startTime, startTime, 0, sd, null,
-        emptyParameters, null, null, null);
-    store.createTable(table);
-
-    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
-
-    // Drop one of the partitions and make sure it doesn't drop the storage descriptor
-    store.dropPartition(dbName, tableName, Arrays.asList(partVals.get(0)));
-    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
-
-    // Alter the second table in a few ways to make sure it changes it's descriptor properly
-    table = store.getTable(dbName, tableName2);
-    byte[] sdHash = HBaseUtils.hashStorageDescriptor(table.getSd(), md);
-
-    // Alter the table without touching the storage descriptor
-    table.setLastAccessTime(startTime + 1);
-    store.alterTable(dbName, tableName2, table);
-    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
-    table = store.getTable(dbName, tableName2);
-    byte[] alteredHash = HBaseUtils.hashStorageDescriptor(table.getSd(), md);
-    Assert.assertArrayEquals(sdHash, alteredHash);
-
-    // Alter the table, changing the storage descriptor
-    table.getSd().setOutputFormat("output_changed");
-    store.alterTable(dbName, tableName2, table);
-    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
-    table = store.getTable(dbName, tableName2);
-    alteredHash = HBaseUtils.hashStorageDescriptor(table.getSd(), md);
-    Assert.assertFalse(Arrays.equals(sdHash, alteredHash));
-
-    // Alter one of the partitions without touching the storage descriptor
-    Partition part = store.getPartition(dbName, tableName, Arrays.asList(partVals.get(1)));
-    sdHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
-    part.setLastAccessTime(part.getLastAccessTime() + 1);
-    store.alterPartition(dbName, tableName, Arrays.asList(partVals.get(1)), part);
-    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
-    part = store.getPartition(dbName, tableName, Arrays.asList(partVals.get(1)));
-    alteredHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
-    Assert.assertArrayEquals(sdHash, alteredHash);
-
-    // Alter the partition, changing the storage descriptor
-    part.getSd().setOutputFormat("output_changed_some_more");
-    store.alterPartition(dbName, tableName, Arrays.asList(partVals.get(1)), part);
-    Assert.assertEquals(3, HBaseReadWrite.getInstance().countStorageDescriptor());
-    part = store.getPartition(dbName, tableName, Arrays.asList(partVals.get(1)));
-    alteredHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
-    Assert.assertFalse(Arrays.equals(sdHash, alteredHash));
-
-    // Alter multiple partitions without touching the storage descriptors
-    List<Partition> parts = store.getPartitions(dbName, tableName, -1);
-    sdHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
-    for (int i = 1; i < 3; i++) {
-      parts.get(i).setLastAccessTime(97);
-    }
-    List<List<String>> listPartVals = new ArrayList<List<String>>();
-    for (String pv : partVals.subList(1, partVals.size())) {
-      listPartVals.add(Arrays.asList(pv));
-    }
-    store.alterPartitions(dbName, tableName, listPartVals, parts);
-    Assert.assertEquals(3, HBaseReadWrite.getInstance().countStorageDescriptor());
-    parts = store.getPartitions(dbName, tableName, -1);
-    alteredHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
-    Assert.assertArrayEquals(sdHash, alteredHash);
-
-    // Alter multiple partitions changning the storage descriptors
-    parts = store.getPartitions(dbName, tableName, -1);
-    sdHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
-    for (int i = 1; i < 3; i++) {
-      parts.get(i).getSd().setOutputFormat("yet_a_different_of");
-    }
-    store.alterPartitions(dbName, tableName, listPartVals, parts);
-    Assert.assertEquals(4, HBaseReadWrite.getInstance().countStorageDescriptor());
-    parts = store.getPartitions(dbName, tableName, -1);
-    alteredHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
-    Assert.assertFalse(Arrays.equals(sdHash, alteredHash));
-
-    for (String partVal : partVals.subList(1, partVals.size())) {
-      store.dropPartition(dbName, tableName, Arrays.asList(partVal));
-    }
-    store.dropTable(dbName, tableName);
-    store.dropTable(dbName, tableName2);
-
-    Assert.assertEquals(0, HBaseReadWrite.getInstance().countStorageDescriptor());
-
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index c17ca10..dd2be96 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -90,7 +90,7 @@ public class TestLocationQueries extends BaseTestQueries {
         String hadoopVer, String locationSubdir)
       throws Exception
     {
-      super(outDir, logDir, miniMr, null, hadoopVer, "", "", false, false);
+      super(outDir, logDir, miniMr, null, hadoopVer, "", "", false);
       this.locationSubdir = locationSubdir;
     }
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
index 749abb5..a4dd07e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
@@ -29,7 +29,7 @@ public class AccumuloQTestUtil extends QTestUtil {
   public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr,
       AccumuloTestSetup setup, String initScript, String cleanupScript) throws Exception {
 
-    super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false, false);
+    super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false);
     setup.setupWithHiveConf(conf);
     this.setup = setup;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
index c12f51e..72336e2 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
@@ -45,7 +45,7 @@ public abstract class AbstractCliConfig {
   public static final String HIVE_ROOT = getHiveRoot();
 
   public static enum MetastoreType {
-    sql, hbase
+    sql
   };
 
   private MetastoreType metastoreType = MetastoreType.sql;
@@ -413,8 +413,6 @@ public abstract class AbstractCliConfig {
     if (metaStoreTypeProperty != null) {
       if (metaStoreTypeProperty.equalsIgnoreCase("sql")) {
         metastoreType = MetastoreType.sql;
-      } else if (metaStoreTypeProperty.equalsIgnoreCase("hbase")) {
-        metastoreType = MetastoreType.hbase;
       } else {
         throw new IllegalArgumentException("Unknown metastore type: " + metaStoreTypeProperty);
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 02abe53..67e03a4 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -55,11 +55,10 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
     String hiveConfDir = cliConfig.getHiveConfDir();
     String initScript = cliConfig.getInitScript();
     String cleanupScript = cliConfig.getCleanupScript();
-    boolean useHBaseMetastore = cliConfig.getMetastoreType() == MetastoreType.hbase;
     try {
       String hadoopVer = cliConfig.getHadoopVersion();
       qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR,
-          hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true);
+          hiveConfDir, hadoopVer, initScript, cleanupScript, true);
 
       if (Strings.isNullOrEmpty(qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH))) {
         fail(String.format("%s must be set. Try setting in blobstore-conf.xml", HCONF_TEST_BLOBSTORE_PATH));

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index d59b650..a1762ec 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -55,7 +55,6 @@ public class CoreCliDriver extends CliAdapter {
     final String hiveConfDir = cliConfig.getHiveConfDir();
     final String initScript = cliConfig.getInitScript();
     final String cleanupScript = cliConfig.getCleanupScript();
-    final boolean useHBaseMetastore = cliConfig.getMetastoreType() == MetastoreType.hbase;
     try {
       final String hadoopVer = cliConfig.getHadoopVersion();
 
@@ -63,8 +62,7 @@ public class CoreCliDriver extends CliAdapter {
         @Override
         public QTestUtil invokeInternal() throws Exception {
           return new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR,
-              hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true,
-              cliConfig.getFsType());
+              hiveConfDir, hadoopVer, initScript, cleanupScript, true, cliConfig.getFsType());
         }
       }.invoke("QtestUtil instance created", LOG, true);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
index bff81dd..64b419b 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
@@ -52,7 +52,7 @@ public class CoreCompareCliDriver extends CliAdapter{
     try {
       String hadoopVer = cliConfig.getHadoopVersion();
       qt = new QTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR,
-      hiveConfDir, hadoopVer, initScript, cleanupScript, false, false);
+      hiveConfDir, hadoopVer, initScript, cleanupScript, false);
 
       // do a one time initialization
       qt.cleanUp();

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 438a61e..7947988 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -45,7 +45,7 @@ public class CoreNegativeCliDriver extends CliAdapter{
     try {
       String hadoopVer = cliConfig.getHadoopVersion();
       qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR,
-       hiveConfDir, hadoopVer, initScript, cleanupScript, false, false);
+       hiveConfDir, hadoopVer, initScript, cleanupScript, false);
       // do a one time initialization
       qt.cleanUp();
       qt.createSources();

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index 34eeb77..d80bd44 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -61,7 +61,7 @@ public class CorePerfCliDriver extends CliAdapter{
       String hadoopVer = cliConfig.getHadoopVersion();
       qt = new QTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, hiveConfDir,
           hadoopVer, initScript,
-          cleanupScript, false, false);
+          cleanupScript, false);
 
       // do a one time initialization
       qt.cleanUp();

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
index 01faaba..aeb7215 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
@@ -46,7 +46,7 @@ public class HBaseQTestUtil extends QTestUtil {
     String initScript, String cleanupScript)
     throws Exception {
 
-    super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false, false);
+    super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false);
     hbaseSetup = setup;
     hbaseSetup.preTest(conf);
     this.conn = setup.getConnection();

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
deleted file mode 100644
index 21e8f7e..0000000
--- a/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-import java.util.List;
-
-public class HBaseStoreTestUtil {
-  public static void initHBaseMetastore(HBaseAdmin admin, HiveConf conf) throws Exception {
-    for (String tableName : HBaseReadWrite.tableNames) {
-      List<byte[]> families = HBaseReadWrite.columnFamilies.get(tableName);
-      HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
-      for (byte[] family : families) {
-        HColumnDescriptor columnDesc = new HColumnDescriptor(family);
-        desc.addFamily(columnDesc);
-      }
-      admin.createTable(desc);
-    }
-    admin.close();
-    if (conf != null) {
-      HBaseReadWrite.setConf(conf);
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 535cfd9..825f826 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -103,7 +103,6 @@ import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
 import org.apache.hadoop.hive.llap.io.api.LlapProxy;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.hbase.HBaseStore;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -202,7 +201,6 @@ public class QTestUtil {
 
   private final String initScript;
   private final String cleanupScript;
-  private boolean useHBaseMetastore = false;
 
   public interface SuiteAddTestFunctor {
     public void addTestToSuite(TestSuite suite, Object setup, String tName);
@@ -347,14 +345,9 @@ public class QTestUtil {
       conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, true);
     }
 
-    if (!useHBaseMetastore) {
-      // Plug verifying metastore in for testing DirectSQL.
-      conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
-          "org.apache.hadoop.hive.metastore.VerifyingObjectStore");
-    } else {
-      conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, HBaseStore.class.getName());
-      conf.setBoolVar(ConfVars.METASTORE_FASTPATH, true);
-    }
+    // Plug verifying metastore in for testing DirectSQL.
+    conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL,
+        "org.apache.hadoop.hive.metastore.VerifyingObjectStore");
 
     if (mr != null) {
       mr.setupConfiguration(conf);
@@ -514,40 +507,22 @@ public class QTestUtil {
     return "jceks://file" + new Path(keyDir, "test.jks").toUri();
   }
 
-  private void startMiniHBaseCluster() throws Exception {
-    Configuration hbaseConf = HBaseConfiguration.create();
-    hbaseConf.setInt("hbase.master.info.port", -1);
-    utility = new HBaseTestingUtility(hbaseConf);
-    utility.startMiniCluster();
-    conf = new HiveConf(utility.getConfiguration(), Driver.class);
-    HBaseAdmin admin = utility.getHBaseAdmin();
-    // Need to use reflection here to make compilation pass since HBaseIntegrationTests
-    // is not compiled in hadoop-1. All HBaseMetastore tests run under hadoop-2, so this
-    // guarantee HBaseIntegrationTests exist when we hitting this code path
-    java.lang.reflect.Method initHBaseMetastoreMethod = Class.forName(
-        "org.apache.hadoop.hive.metastore.hbase.HBaseStoreTestUtil")
-        .getMethod("initHBaseMetastore", HBaseAdmin.class, HiveConf.class);
-    initHBaseMetastoreMethod.invoke(null, admin, conf);
-    conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, HBaseStore.class.getName());
-    conf.setBoolVar(ConfVars.METASTORE_FASTPATH, true);
-  }
-
   public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
                    String confDir, String hadoopVer, String initScript, String cleanupScript,
-                   boolean useHBaseMetastore, boolean withLlapIo) throws Exception {
+                   boolean withLlapIo) throws Exception {
     this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
-        useHBaseMetastore, withLlapIo, null);
+        withLlapIo, null);
   }
 
   public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
       String confDir, String hadzoopVer, String initScript, String cleanupScript,
-      boolean useHBaseMetastore, boolean withLlapIo, FsType fsType)
+      boolean withLlapIo, FsType fsType)
     throws Exception {
     LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," +
-        " hadoopVer={}, initScript={}, cleanupScript={}, useHbaseMetaStore={}, withLlapIo={}," +
+        " hadoopVer={}, initScript={}, cleanupScript={}, withLlapIo={}," +
             " fsType={}"
         , outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
-        useHBaseMetastore, withLlapIo, fsType);
+        withLlapIo, fsType);
     Preconditions.checkNotNull(clusterType, "ClusterType cannot be null");
     if (fsType != null) {
       this.fsType = fsType;
@@ -556,7 +531,6 @@ public class QTestUtil {
     }
     this.outDir = outDir;
     this.logDir = logDir;
-    this.useHBaseMetastore = useHBaseMetastore;
     this.srcTables=getSrcTables();
     this.srcUDFs = getSrcUDFs();
 
@@ -567,11 +541,7 @@ public class QTestUtil {
     }
 
     queryState = new QueryState.Builder().withHiveConf(new HiveConf(Driver.class)).build();
-    if (useHBaseMetastore) {
-      startMiniHBaseCluster();
-    } else {
-      conf = queryState.getConf();
-    }
+    conf = queryState.getConf();
     this.hadoopVer = getHadoopMainVersion(hadoopVer);
     qMap = new TreeMap<String, String>();
     qSkipSet = new HashSet<String>();
@@ -697,9 +667,6 @@ public class QTestUtil {
         sparkSession = null;
       }
     }
-    if (useHBaseMetastore) {
-      utility.shutdownMiniCluster();
-    }
     if (mr != null) {
       mr.shutdown();
       mr = null;
@@ -2032,8 +1999,7 @@ public class QTestUtil {
     for (int i = 0; i < qfiles.length; i++) {
       qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20",
         initScript == null ? defaultInitScript : initScript,
-        cleanupScript == null ? defaultCleanupScript : cleanupScript,
-        false, false);
+        cleanupScript == null ? defaultCleanupScript : cleanupScript, false);
       qt[i].addFile(qfiles[i]);
       qt[i].clearTestSideEffects();
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index 31f69a3..30ac6d1 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -54,8 +54,7 @@ public class CoreParseNegative extends CliAdapter{
     try {
       String hadoopVer = cliConfig.getHadoopVersion();
       qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR, null,
-          hadoopVer,
-       initScript, cleanupScript, false, false);
+          hadoopVer, initScript, cleanupScript, false);
     } catch (Exception e) {
       System.err.println("Exception: " + e.getMessage());
       e.printStackTrace();

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/pom.xml
----------------------------------------------------------------------
diff --git a/metastore/pom.xml b/metastore/pom.xml
index 073d039..1fa17a7 100644
--- a/metastore/pom.xml
+++ b/metastore/pom.xml
@@ -304,11 +304,6 @@
                     <mkdir dir="${protobuf.build.dir}"/>
                     <exec executable="protoc" failonerror="true">
                       <arg value="--java_out=${protobuf.build.dir}"/>
-                      <arg value="-I=${protobuf.src.dir}/org/apache/hadoop/hive/metastore/hbase"/>
-                      <arg value="${protobuf.src.dir}/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto"/>
-                    </exec>
-                    <exec executable="protoc" failonerror="true">
-                      <arg value="--java_out=${protobuf.build.dir}"/>
                       <arg value="-I=${protobuf.src.dir}/org/apache/hadoop/hive/metastore"/>
                       <arg value="${protobuf.src.dir}/org/apache/hadoop/hive/metastore/metastore.proto"/>
                     </exec>


[04/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
deleted file mode 100644
index c6a134c..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHBaseAggregateStatsCache {
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseAggregateStatsCache.class.getName());
-
-  @Mock HTableInterface htable;
-  private HBaseStore store;
-  SortedMap<String, Cell> rows = new TreeMap<>();
-
-  @Before
-  public void before() throws IOException {
-    MockitoAnnotations.initMocks(this);
-    HiveConf conf = new HiveConf();
-    conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
-    store = MockUtils.init(conf, htable, rows);
-    store.backdoor().getStatsCache().resetCounters();
-  }
-
-  private static interface Checker {
-    void checkStats(AggrStats aggrStats) throws Exception;
-  }
-
-  // Do to limitations in the Mock infrastructure we use for HBase testing we can only test
-  // this for a single column table and we can't really test hits in hbase, only in memory or
-  // build from scratch.  But it's still useful to cover many bugs.  More in depth testing with
-  // multiple columns and with HBase hits is done in TestHBaseAggrStatsCacheIntegration.
-
-  @Test
-  public void allWithStats() throws Exception {
-    String dbName = "default";
-    String tableName = "hit";
-    List<String> partVals1 = Arrays.asList("today");
-    List<String> partVals2 = Arrays.asList("yesterday");
-    long now = System.currentTimeMillis();
-
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1", "boolean", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String>emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
-        Collections.<String, String>emptyMap(), null, null, null);
-    store.createTable(table);
-
-    for (List<String> partVals : Arrays.asList(partVals1, partVals2)) {
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVals.get(0));
-      Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String>emptyMap());
-      store.addPartition(part);
-
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVals.get(0));
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col1");
-      obj.setColType("boolean");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
-      bcsd.setNumFalses(10);
-      bcsd.setNumTrues(20);
-      bcsd.setNumNulls(30);
-      data.setBooleanStats(bcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-
-      store.updatePartitionColumnStatistics(cs, partVals);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(2, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1", cso.getColName());
-        Assert.assertEquals("boolean", cso.getColType());
-        BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
-        Assert.assertEquals(20, bcsd.getNumFalses());
-        Assert.assertEquals(40, bcsd.getNumTrues());
-        Assert.assertEquals(60, bcsd.getNumNulls());
-      }
-    };
-
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    // Check that we had to build it from the stats
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
-    // Call again, this time it should come from memory.  Also, reverse the name order this time
-    // to assure that we still hit.
-    aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-  }
-
-
-  @Test
-  public void noneWithStats() throws Exception {
-    String dbName = "default";
-    String tableName = "nws";
-    List<String> partVals1 = Arrays.asList("today");
-    List<String> partVals2 = Arrays.asList("yesterday");
-    long now = System.currentTimeMillis();
-
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1", "boolean", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String>emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
-        Collections.<String, String>emptyMap(), null, null, null);
-    store.createTable(table);
-
-    for (List<String> partVals : Arrays.asList(partVals1, partVals2)) {
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/nws/ds=" + partVals.get(0));
-      Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String>emptyMap());
-      store.addPartition(part);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(0, aggrStats.getPartsFound());
-      }
-    };
-
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void someNonexistentPartitions() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    List<String> partVals1 = Arrays.asList("today");
-    List<String> partVals2 = Arrays.asList("yesterday");
-    long now = System.currentTimeMillis();
-
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1", "boolean", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String>emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
-        Collections.<String, String>emptyMap(), null, null, null);
-    store.createTable(table);
-
-    StorageDescriptor psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/default/hit/ds=" + partVals1.get(0));
-    Partition part = new Partition(partVals1, dbName, tableName, (int) now, (int) now, psd,
-        Collections.<String, String>emptyMap());
-    store.addPartition(part);
-
-    ColumnStatistics cs = new ColumnStatistics();
-    ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-    desc.setLastAnalyzed(now);
-    desc.setPartName("ds=" + partVals1.get(0));
-    cs.setStatsDesc(desc);
-    ColumnStatisticsObj obj = new ColumnStatisticsObj();
-    obj.setColName("col1");
-    obj.setColType("double");
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
-    dcsd.setHighValue(1000.2342343);
-    dcsd.setLowValue(-20.1234213423);
-    dcsd.setNumNulls(30);
-    dcsd.setNumDVs(12342);
-    data.setDoubleStats(dcsd);
-    obj.setStatsData(data);
-    cs.addToStatsObj(obj);
-
-    store.updatePartitionColumnStatistics(cs, partVals1);
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(1, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1", cso.getColName());
-        Assert.assertEquals("double", cso.getColType());
-        DoubleColumnStatsData dcsd = cso.getStatsData().getDoubleStats();
-        Assert.assertEquals(1000.23, dcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-20.12, dcsd.getLowValue(), 0.01);
-        Assert.assertEquals(30, dcsd.getNumNulls());
-        Assert.assertEquals(12342, dcsd.getNumDVs());
-      }
-    };
-
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    // Check that we had to build it from the stats
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
-    // Call again, this time it should come from memory.  Also, reverse the name order this time
-    // to assure that we still hit.
-    aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-  }
-
-  @Test
-  public void nonexistentPartitions() throws Exception {
-    String dbName = "default";
-    String tableName = "nep";
-    List<String> partVals1 = Arrays.asList("today");
-    List<String> partVals2 = Arrays.asList("yesterday");
-    long now = System.currentTimeMillis();
-
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1", "boolean", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String>emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
-        Collections.<String, String>emptyMap(), null, null, null);
-    store.createTable(table);
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(0, aggrStats.getPartsFound());
-      }
-    };
-
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    // Check that we had to build it from the stats
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-  }
-  // TODO test invalidation
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
deleted file mode 100644
index 9cf1fb8..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHBaseAggregateStatsCacheWithBitVector {
-  private static final Logger LOG = LoggerFactory
-      .getLogger(TestHBaseAggregateStatsCacheWithBitVector.class.getName());
-
-  @Mock
-  HTableInterface htable;
-  private HBaseStore store;
-  SortedMap<String, Cell> rows = new TreeMap<>();
-
-  @Before
-  public void before() throws IOException {
-    MockitoAnnotations.initMocks(this);
-    HiveConf conf = new HiveConf();
-    conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
-    store = MockUtils.init(conf, htable, rows);
-    store.backdoor().getStatsCache().resetCounters();
-  }
-
-  private static interface Checker {
-    void checkStats(AggrStats aggrStats) throws Exception;
-  }
-
-  @Test
-  public void allPartitions() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    List<String> partVals1 = Arrays.asList("today");
-    List<String> partVals2 = Arrays.asList("yesterday");
-    long now = System.currentTimeMillis();
-
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1", "boolean", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    StorageDescriptor psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/default/hit/ds=" + partVals1.get(0));
-    Partition part = new Partition(partVals1, dbName, tableName, (int) now, (int) now, psd,
-        Collections.<String, String> emptyMap());
-    store.addPartition(part);
-
-    psd = new StorageDescriptor(sd);
-    psd.setLocation("file:/tmp/default/hit/ds=" + partVals2.get(0));
-    part = new Partition(partVals2, dbName, tableName, (int) now, (int) now, psd,
-        Collections.<String, String> emptyMap());
-    store.addPartition(part);
-
-    ColumnStatistics cs = new ColumnStatistics();
-    ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-    desc.setLastAnalyzed(now);
-    desc.setPartName("ds=" + partVals1.get(0));
-    cs.setStatsDesc(desc);
-    ColumnStatisticsObj obj = new ColumnStatisticsObj();
-    obj.setColName("col1");
-    obj.setColType("double");
-    ColumnStatisticsData data = new ColumnStatisticsData();
-    DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
-    dcsd.setHighValue(1000.2342343);
-    dcsd.setLowValue(-20.1234213423);
-    dcsd.setNumNulls(30);
-    dcsd.setNumDVs(12342);
-    HyperLogLog hll = HyperLogLog.builder().build();
-    hll.addDouble(1);
-    hll.addDouble(2);
-    hll.addDouble(3);
-    dcsd.setBitVectors(hll.serialize());
-    data.setDoubleStats(dcsd);
-    obj.setStatsData(data);
-    cs.addToStatsObj(obj);
-    store.updatePartitionColumnStatistics(cs, partVals1);
-
-    cs = new ColumnStatistics();
-    desc = new ColumnStatisticsDesc(false, dbName, tableName);
-    desc.setLastAnalyzed(now);
-    desc.setPartName("ds=" + partVals2.get(0));
-    cs.setStatsDesc(desc);
-    obj = new ColumnStatisticsObj();
-    obj.setColName("col1");
-    obj.setColType("double");
-    data = new ColumnStatisticsData();
-    dcsd = new DoubleColumnStatsData();
-    dcsd.setHighValue(1000.2342343);
-    dcsd.setLowValue(-20.1234213423);
-    dcsd.setNumNulls(30);
-    dcsd.setNumDVs(12342);
-    hll = HyperLogLog.builder().build();
-    hll.addDouble(3);
-    hll.addDouble(4);
-    hll.addDouble(5);
-    dcsd.setBitVectors(hll.serialize());
-    data.setDoubleStats(dcsd);
-    obj.setStatsData(data);
-    cs.addToStatsObj(obj);
-
-    store.updatePartitionColumnStatistics(cs, partVals2);
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(2, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1", cso.getColName());
-        Assert.assertEquals("double", cso.getColType());
-        DoubleColumnStatsData dcsd = cso.getStatsData().getDoubleStats();
-        Assert.assertEquals(1000.23, dcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-20.12, dcsd.getLowValue(), 0.01);
-        Assert.assertEquals(60, dcsd.getNumNulls());
-        Assert.assertEquals(5, dcsd.getNumDVs());
-        // we do not store the bitvector for the aggrStats.
-        // we can store that if it is necessary in the future.
-      }
-    };
-
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    // Check that we had to build it from the stats
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
-    // Call again, this time it should come from memory. Also, reverse the name
-    // order this time
-    // to assure that we still hit.
-    aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
deleted file mode 100644
index 4d868b0..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
+++ /dev/null
@@ -1,721 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.StatObjectConverter;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHBaseAggregateStatsExtrapolation {
-  private static final Logger LOG = LoggerFactory
-      .getLogger(TestHBaseAggregateStatsExtrapolation.class.getName());
-
-  @Mock
-  HTableInterface htable;
-  private HBaseStore store;
-  SortedMap<String, Cell> rows = new TreeMap<>();
-
-  // NDV will be 3 for the bitVectors
-  String bitVectors = null;
-  @Before
-  public void before() throws IOException {
-    MockitoAnnotations.initMocks(this);
-    HiveConf conf = new HiveConf();
-    conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
-    store = MockUtils.init(conf, htable, rows);
-    store.backdoor().getStatsCache().resetCounters();
-    HyperLogLog hll = HyperLogLog.builder().build();
-    hll.addLong(1);
-    hll.addLong(2);
-    hll.addLong(3);
-    bitVectors = hll.serialize();
-  }
-
-  private static interface Checker {
-    void checkStats(AggrStats aggrStats) throws Exception;
-  }
-
-  @Test
-  public void allPartitionsHaveBitVectorStatusLong() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVal);
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col1");
-      obj.setColType("long");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      LongColumnStatsData dcsd = new LongColumnStatsData();
-      dcsd.setHighValue(1000 + i);
-      dcsd.setLowValue(-1000 - i);
-      dcsd.setNumNulls(i);
-      dcsd.setNumDVs(10 * i + 1);
-      dcsd.setBitVectors(bitVectors);
-      data.setLongStats(dcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-      store.updatePartitionColumnStatistics(cs, partVal);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(10, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void allPartitionsHaveBitVectorStatusDecimal() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1_decimal", "decimal", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVal);
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col1_decimal");
-      obj.setColType("decimal");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      DecimalColumnStatsData dcsd = new DecimalColumnStatsData();
-      dcsd.setHighValue(StatObjectConverter.createThriftDecimal("" + (1000 + i)));
-      dcsd.setLowValue(StatObjectConverter.createThriftDecimal("" + (-1000 - i)));
-      dcsd.setNumNulls(i);
-      dcsd.setNumDVs(10 * i + 1);
-      dcsd.setBitVectors(bitVectors);
-      data.setDecimalStats(dcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-      store.updatePartitionColumnStatistics(cs, partVal);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(10, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1_decimal", cso.getColName());
-        Assert.assertEquals("decimal", cso.getColType());
-        DecimalColumnStatsData lcsd = cso.getStatsData().getDecimalStats();
-        Assert.assertEquals(1009, HBaseUtils.getDoubleValue(lcsd.getHighValue()), 0.01);
-        Assert.assertEquals(-1009, HBaseUtils.getDoubleValue(lcsd.getLowValue()), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col1_decimal"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void allPartitionsHaveBitVectorStatusDouble() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1_double", "double", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVal);
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col1_double");
-      obj.setColType("double");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
-      dcsd.setHighValue(1000 + i);
-      dcsd.setLowValue(-1000 - i);
-      dcsd.setNumNulls(i);
-      dcsd.setNumDVs(10 * i + 1);
-      dcsd.setBitVectors(bitVectors);
-      data.setDoubleStats(dcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-      store.updatePartitionColumnStatistics(cs, partVal);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(10, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1_double", cso.getColName());
-        Assert.assertEquals("double", cso.getColType());
-        DoubleColumnStatsData lcsd = cso.getStatsData().getDoubleStats();
-        Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col1_double"));
-    statChecker.checkStats(aggrStats);
-  }
-  
-  @Test
-  public void allPartitionsHaveBitVectorStatusString() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1_string", "string", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVal);
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col1_string");
-      obj.setColType("string");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      StringColumnStatsData dcsd = new StringColumnStatsData();
-      dcsd.setAvgColLen(i + 1);
-      dcsd.setMaxColLen(i + 10);
-      dcsd.setNumNulls(i);
-      dcsd.setNumDVs(10 * i + 1);
-      dcsd.setBitVectors(bitVectors);
-      data.setStringStats(dcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-      store.updatePartitionColumnStatistics(cs, partVal);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(10, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1_string", cso.getColName());
-        Assert.assertEquals("string", cso.getColType());
-        StringColumnStatsData lcsd = cso.getStatsData().getStringStats();
-        Assert.assertEquals(10, lcsd.getAvgColLen(), 0.01);
-        Assert.assertEquals(19, lcsd.getMaxColLen(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col1_string"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void noPartitionsHaveBitVectorStatus() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col2", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVal);
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col2");
-      obj.setColType("long");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      LongColumnStatsData dcsd = new LongColumnStatsData();
-      dcsd.setHighValue(1000 + i);
-      dcsd.setLowValue(-1000 - i);
-      dcsd.setNumNulls(i);
-      dcsd.setNumDVs(i == 0 ? 1 : 10 * i);
-      data.setLongStats(dcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-      store.updatePartitionColumnStatistics(cs, partVal);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(10, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col2", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(90, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col2"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void TwoEndsOfPartitionsHaveBitVectorStatus() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col3", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      if (i < 2 || i > 7) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVal);
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col3");
-        obj.setColType("long");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        LongColumnStatsData dcsd = new LongColumnStatsData();
-        dcsd.setHighValue(1000 + i);
-        dcsd.setLowValue(-1000 - i);
-        dcsd.setNumNulls(i);
-        dcsd.setNumDVs(10 * i);
-        dcsd.setBitVectors(bitVectors);
-        data.setLongStats(dcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-        store.updatePartitionColumnStatistics(cs, partVal);
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(4, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col3", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col3"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void MiddleOfPartitionsHaveBitVectorStatus() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col4", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      if (i > 2 && i < 7) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVal);
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col4");
-        obj.setColType("long");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        LongColumnStatsData dcsd = new LongColumnStatsData();
-        dcsd.setHighValue(1000 + i);
-        dcsd.setLowValue(-1000 - i);
-        dcsd.setNumNulls(i);
-        dcsd.setNumDVs(10 * i);
-        dcsd.setBitVectors(bitVectors);
-        data.setLongStats(dcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-        store.updatePartitionColumnStatistics(cs, partVal);
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(4, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col4", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1006, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1006, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col4"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col5", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVal);
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col5");
-        obj.setColType("long");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        LongColumnStatsData dcsd = new LongColumnStatsData();
-        dcsd.setHighValue(1000 + i);
-        dcsd.setLowValue(-1000 - i);
-        dcsd.setNumNulls(i);
-        dcsd.setNumDVs(10 * i);
-        dcsd.setBitVectors(bitVectors);
-        data.setLongStats(dcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-        store.updatePartitionColumnStatistics(cs, partVal);
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(6, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col5", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(40, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col5"));
-    statChecker.checkStats(aggrStats);
-  }
-  
-  @Test
-  public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusDouble() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col5_double", "double", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVal);
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col5_double");
-        obj.setColType("double");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
-        dcsd.setHighValue(1000 + i);
-        dcsd.setLowValue(-1000 - i);
-        dcsd.setNumNulls(i);
-        dcsd.setNumDVs(10 * i);
-        dcsd.setBitVectors(bitVectors);
-        data.setDoubleStats(dcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-        store.updatePartitionColumnStatistics(cs, partVal);
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(6, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col5_double", cso.getColName());
-        Assert.assertEquals("double", cso.getColType());
-        DoubleColumnStatsData lcsd = cso.getStatsData().getDoubleStats();
-        Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(40, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col5_double"));
-    statChecker.checkStats(aggrStats);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
deleted file mode 100644
index 0ad2780..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
+++ /dev/null
@@ -1,589 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.StatObjectConverter;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHBaseAggregateStatsNDVUniformDist {
-  private static final Logger LOG = LoggerFactory
-      .getLogger(TestHBaseAggregateStatsNDVUniformDist.class.getName());
-
-  @Mock
-  HTableInterface htable;
-  private HBaseStore store;
-  SortedMap<String, Cell> rows = new TreeMap<>();
-
-  // NDV will be 3 for bitVectors[0] and 1 for bitVectors[1]
-  String bitVectors[] = new String[2];
-    
-
-  @Before
-  public void before() throws IOException {
-    MockitoAnnotations.initMocks(this);
-    HiveConf conf = new HiveConf();
-    conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
-    conf.setBoolean(HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION.varname, true);
-    store = MockUtils.init(conf, htable, rows);
-    store.backdoor().getStatsCache().resetCounters();
-    HyperLogLog hll = HyperLogLog.builder().build();
-    hll.addLong(1);
-    bitVectors[1] = hll.serialize();
-    hll = HyperLogLog.builder().build();
-    hll.addLong(2);
-    hll.addLong(3);
-    hll.addLong(3);
-    hll.addLong(4);
-    bitVectors[0] = hll.serialize();
-  }
-
-  private static interface Checker {
-    void checkStats(AggrStats aggrStats) throws Exception;
-  }
-
-  @Test
-  public void allPartitionsHaveBitVectorStatus() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVal);
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col1");
-      obj.setColType("long");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      LongColumnStatsData dcsd = new LongColumnStatsData();
-      dcsd.setHighValue(1000 + i);
-      dcsd.setLowValue(-1000 - i);
-      dcsd.setNumNulls(i);
-      dcsd.setNumDVs(10 * i + 1);
-      dcsd.setBitVectors(bitVectors[0]);
-      data.setLongStats(dcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-      store.updatePartitionColumnStatistics(cs, partVal);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(10, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void noPartitionsHaveBitVectorStatus() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col2", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVal);
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col2");
-      obj.setColType("long");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      LongColumnStatsData dcsd = new LongColumnStatsData();
-      dcsd.setHighValue(1000 + i);
-      dcsd.setLowValue(-1000 - i);
-      dcsd.setNumNulls(i);
-      dcsd.setNumDVs(10 * i + 1);
-      data.setLongStats(dcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-      store.updatePartitionColumnStatistics(cs, partVal);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(10, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col2", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(91, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col2"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void TwoEndsOfPartitionsHaveBitVectorStatus() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col3", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      if (i < 2 || i > 7) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVal);
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col3");
-        obj.setColType("long");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        LongColumnStatsData dcsd = new LongColumnStatsData();
-        dcsd.setHighValue(1000 + i);
-        dcsd.setLowValue(-1000 - i);
-        dcsd.setNumNulls(i);
-        dcsd.setNumDVs(10 * i + 1);
-        dcsd.setBitVectors(bitVectors[i / 5]);
-        data.setLongStats(dcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-        store.updatePartitionColumnStatistics(cs, partVal);
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(4, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col3", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col3"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void MiddleOfPartitionsHaveBitVectorStatus() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col4", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      if (i > 2 && i < 7) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVal);
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col4");
-        obj.setColType("long");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        LongColumnStatsData dcsd = new LongColumnStatsData();
-        dcsd.setHighValue(1000 + i);
-        dcsd.setLowValue(-1000 - i);
-        dcsd.setNumNulls(i);
-        dcsd.setNumDVs(10 * i + 1);
-        dcsd.setBitVectors(bitVectors[0]);
-        data.setLongStats(dcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-        store.updatePartitionColumnStatistics(cs, partVal);
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(4, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col4", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1006, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1006, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(45, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col4"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col5_long", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVal);
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col5_long");
-        obj.setColType("long");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        LongColumnStatsData dcsd = new LongColumnStatsData();
-        dcsd.setHighValue(1000 + i);
-        dcsd.setLowValue(-1000 - i);
-        dcsd.setNumNulls(i);
-        dcsd.setNumDVs(10 * i + 1);
-        dcsd.setBitVectors(bitVectors[i / 5]);
-        data.setLongStats(dcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-        store.updatePartitionColumnStatistics(cs, partVal);
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(6, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col5_long", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(40, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col5_long"));
-    statChecker.checkStats(aggrStats);
-  }
-  
-  @Test
-  public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusDecimal() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col5_decimal", "decimal", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVal);
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col5_decimal");
-        obj.setColType("decimal");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        DecimalColumnStatsData dcsd = new DecimalColumnStatsData();
-        dcsd.setHighValue(StatObjectConverter.createThriftDecimal("" + (1000 + i)));
-        dcsd.setLowValue(StatObjectConverter.createThriftDecimal("" + (-1000 - i)));
-        dcsd.setNumNulls(i);
-        dcsd.setNumDVs(10 * i + 1);
-        dcsd.setBitVectors(bitVectors[i / 5]);
-        data.setDecimalStats(dcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-        store.updatePartitionColumnStatistics(cs, partVal);
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(6, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col5_decimal", cso.getColName());
-        Assert.assertEquals("decimal", cso.getColType());
-        DecimalColumnStatsData lcsd = cso.getStatsData().getDecimalStats();
-        Assert.assertEquals(1010, HBaseUtils.getDoubleValue(lcsd.getHighValue()), 0.01);
-        Assert.assertEquals(-1010, HBaseUtils.getDoubleValue(lcsd.getLowValue()), 0.01);
-        Assert.assertEquals(40, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col5_decimal"));
-    statChecker.checkStats(aggrStats);
-  }
-
-  @Test
-  public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusDouble() throws Exception {
-    String dbName = "default";
-    String tableName = "snp";
-    long now = System.currentTimeMillis();
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col5_double", "double", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String> emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-        Collections.<String, String> emptyMap(), null, null, null);
-    store.createTable(table);
-
-    List<List<String>> partVals = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      List<String> partVal = Arrays.asList("" + i);
-      partVals.add(partVal);
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
-      Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String> emptyMap());
-      store.addPartition(part);
-      if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVal);
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col5_double");
-        obj.setColType("double");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
-        dcsd.setHighValue(1000 + i);
-        dcsd.setLowValue(-1000 - i);
-        dcsd.setNumNulls(i);
-        dcsd.setNumDVs(10 * i + 1);
-        dcsd.setBitVectors(bitVectors[i / 5]);
-        data.setDoubleStats(dcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-        store.updatePartitionColumnStatistics(cs, partVal);
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(6, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col5_double", cso.getColName());
-        Assert.assertEquals("double", cso.getColType());
-        DoubleColumnStatsData lcsd = cso.getStatsData().getDoubleStats();
-        Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
-        Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
-        Assert.assertEquals(40, lcsd.getNumNulls());
-        Assert.assertEquals(3, lcsd.getNumDVs());
-      }
-    };
-    List<String> partNames = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      partNames.add("ds=" + i);
-    }
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
-        Arrays.asList("col5_double"));
-    statChecker.checkStats(aggrStats);
-  }
-}


[06/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionCache.java
deleted file mode 100644
index 08d060f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionCache.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.hive.common.ObjectPair;
-import org.apache.hadoop.hive.metastore.api.Partition;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A cache for partition objects.  This is separate from
- * {@link org.apache.hadoop.hive.metastore.hbase.ObjectCache} because we need to access it
- * differently (always by table) and because we need to be able to track whether we are caching
- * all of the partitions for a table or not.  Like ObjectCache it is local to a particular thread
- * and thus not synchronized.  Also like ObjectCache it is intended to be flushed before each query.
- */
-class PartitionCache {
-  // This is a trie.  The key to the first map is (dbname, tablename), since partitions are
-  // always accessed within the context of the table they belong to.  The second map maps
-  // partition values (not names) to partitions.
-  private Map<ObjectPair<String, String>, TrieValue> cache;
-  private final int maxSize;
-  private int cacheSize;
-  private Counter misses;
-  private Counter hits;
-  private Counter overflows;
-
-  /**
-   *
-   * @param max maximum number of objects to store in the cache.  When max is reached, eviction
-   *            policy is MRU.
-   * @param hits counter to increment when we find an element in the cache
-   * @param misses counter to increment when we do not find an element in the cache
-   * @param overflows counter to increment when we do not have room for an element in the cache
-   */
-  PartitionCache(int max, Counter hits, Counter misses, Counter overflows) {
-    maxSize = max;
-    cache = new HashMap<ObjectPair<String, String>, TrieValue>();
-    cacheSize = 0;
-    this.hits = hits;
-    this.misses = misses;
-    this.overflows = overflows;
-  }
-
-  /**
-   * Put a single partition into the cache
-   * @param dbName
-   * @param tableName
-   * @param part
-   */
-  void put(String dbName, String tableName, Partition part) {
-    if (cacheSize < maxSize) {
-      ObjectPair<String, String> key = new ObjectPair<String, String>(dbName, tableName);
-      TrieValue entry = cache.get(key);
-      if (entry == null) {
-        entry = new TrieValue(false);
-        cache.put(key, entry);
-      }
-      entry.map.put(part.getValues(), part);
-      cacheSize++;
-    } else {
-      overflows.incr();
-    }
-  }
-
-  /**
-   *
-   * @param dbName
-   * @param tableName
-   * @param parts
-   * @param allForTable if true indicates that all partitions for this table are present
-   */
-  void put(String dbName, String tableName, List<Partition> parts, boolean allForTable) {
-    if (cacheSize + parts.size() < maxSize) {
-      ObjectPair<String, String> key = new ObjectPair<String, String>(dbName, tableName);
-      TrieValue entry = cache.get(key);
-      if (entry == null) {
-        entry = new TrieValue(allForTable);
-        cache.put(key, entry);
-      }
-      for (Partition part : parts) entry.map.put(part.getValues(), part);
-      cacheSize += parts.size();
-    } else {
-      overflows.incr();
-    }
-  }
-
-  /**
-   * Will only return a value if all partitions for this table are in the cache.  Otherwise you
-   * should call {@link #get} individually
-   * @param dbName
-   * @param tableName
-   * @return
-   */
-  Collection<Partition> getAllForTable(String dbName, String tableName) {
-    TrieValue entry = cache.get(new ObjectPair<String, String>(dbName, tableName));
-    if (entry != null && entry.hasAllPartitionsForTable) {
-      hits.incr();
-      return entry.map.values();
-    } else {
-      misses.incr();
-      return null;
-    }
-  }
-
-  Partition get(String dbName, String tableName, List<String> partVals) {
-    TrieValue entry = cache.get(new ObjectPair<String, String>(dbName, tableName));
-    if (entry != null) {
-      hits.incr();
-      return entry.map.get(partVals);
-    } else {
-      misses.incr();
-      return null;
-    }
-  }
-
-  void remove(String dbName, String tableName) {
-    ObjectPair<String, String> key = new ObjectPair<String, String>(dbName, tableName);
-    TrieValue entry = cache.get(key);
-    if (entry != null) {
-      cacheSize -= entry.map.size();
-      cache.remove(key);
-    }
-  }
-
-  void remove(String dbName, String tableName, List<String> partVals) {
-    ObjectPair<String, String> key = new ObjectPair<String, String>(dbName, tableName);
-    TrieValue entry = cache.get(key);
-    if (entry != null && entry.map.remove(partVals) != null) {
-      cacheSize--;
-      entry.hasAllPartitionsForTable = false;
-    }
-  }
-
-  void flush() {
-    cache.clear();
-    cacheSize = 0;
-  }
-
-  static class TrieValue {
-    boolean hasAllPartitionsForTable;
-    Map<List<String>, Partition> map;
-
-    TrieValue(boolean hasAll) {
-      hasAllPartitionsForTable = hasAll;
-      map = new HashMap<List<String>, Partition>();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionKeyComparator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionKeyComparator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionKeyComparator.java
deleted file mode 100644
index 2b0863d..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionKeyComparator.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.filter.ByteArrayComparable;
-import org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.PartitionKeyComparator.Operator.Type;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.BytesWritable;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-
-public class PartitionKeyComparator extends ByteArrayComparable {
-  private static final Logger LOG = LoggerFactory.getLogger(PartitionKeyComparator.class);
-  static class Mark {
-    Mark(String value, boolean inclusive) {
-      this.value = value;
-      this.inclusive = inclusive;
-    }
-    String value;
-    boolean inclusive;
-    public String toString() {
-      return value + (inclusive?"_":"");
-    }
-  }
-  static class Range {
-    Range(String keyName, Mark start, Mark end) {
-      this.keyName = keyName;
-      this.start = start;
-      this.end = end;
-    }
-    String keyName;
-    Mark start;
-    Mark end;
-    public String toString() {
-      return "" + keyName + ":" + (start!=null?start.toString():"") + (end!=null?end.toString():"");
-    }
-  }
-  // Cache the information derived from ranges for performance, including
-  // range in native datatype
-  static class NativeRange {
-    int pos;
-    Comparable start;
-    Comparable end;
-  }
-  static class Operator {
-    public Operator(Type type, String keyName, String val) {
-      this.type = type;
-      this.keyName = keyName;
-      this.val = val;
-    }
-    enum Type {
-      LIKE, NOTEQUALS
-    };
-    Type type;
-    String keyName;
-    String val;
-  }
-  static class NativeOperator {
-    int pos;
-    Comparable val;
-  }
-  String names;
-  String types;
-  List<Range> ranges;
-  List<NativeRange> nativeRanges;
-  List<Operator> ops;
-  List<NativeOperator> nativeOps;
-  Properties serdeProps;
-  public PartitionKeyComparator(String names, String types, List<Range> ranges, List<Operator> ops) {
-    super(null);
-    this.names = names;
-    this.types = types;
-    this.ranges = ranges;
-    this.ops = ops;
-    serdeProps = new Properties();
-    serdeProps.setProperty(serdeConstants.LIST_COLUMNS, "dbName,tableName," + names);
-    serdeProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string," + types);
-
-    this.nativeRanges = new ArrayList<NativeRange>(this.ranges.size());
-    for (int i=0;i<ranges.size();i++) {
-      Range range = ranges.get(i);
-      NativeRange nativeRange = new NativeRange();;
-      nativeRanges.add(i, nativeRange);
-      nativeRange.pos = Arrays.asList(names.split(",")).indexOf(range.keyName);
-      TypeInfo expectedType =
-          TypeInfoUtils.getTypeInfoFromTypeString(types.split(",")[nativeRange.pos]);
-      ObjectInspector outputOI =
-          TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
-      nativeRange.start = null;
-      if (range.start != null) {
-        Converter converter = ObjectInspectorConverters.getConverter(
-            PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
-        nativeRange.start = (Comparable)converter.convert(range.start.value);
-      }
-      nativeRange.end = null;
-      if (range.end != null) {
-        Converter converter = ObjectInspectorConverters.getConverter(
-            PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
-        nativeRange.end = (Comparable)converter.convert(range.end.value);
-      }
-    }
-
-    this.nativeOps = new ArrayList<NativeOperator>(this.ops.size());
-    for (int i=0;i<ops.size();i++) {
-      Operator op = ops.get(i);
-      NativeOperator nativeOp = new NativeOperator();
-      nativeOps.add(i, nativeOp);
-      nativeOp.pos = ArrayUtils.indexOf(names.split(","), op.keyName);
-      TypeInfo expectedType =
-          TypeInfoUtils.getTypeInfoFromTypeString(types.split(",")[nativeOp.pos]);
-      ObjectInspector outputOI =
-          TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
-      Converter converter = ObjectInspectorConverters.getConverter(
-          PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
-      nativeOp.val = (Comparable)converter.convert(op.val);
-    }
-  }
-
-  public static PartitionKeyComparator parseFrom(final byte [] bytes) {
-    HbaseMetastoreProto.PartitionKeyComparator proto;
-    try {
-      proto = HbaseMetastoreProto.PartitionKeyComparator.parseFrom(bytes);
-    } catch (InvalidProtocolBufferException e) {
-      throw new RuntimeException(e);
-    }
-    List<Range> ranges = new ArrayList<Range>();
-    for (HbaseMetastoreProto.PartitionKeyComparator.Range range : proto.getRangeList()) {
-      Mark start = null;
-      if (range.hasStart()) {
-        start = new Mark(range.getStart().getValue(), range.getStart().getInclusive());
-      }
-      Mark end = null;
-      if (range.hasEnd()) {
-        end = new Mark(range.getEnd().getValue(), range.getEnd().getInclusive());
-      }
-      ranges.add(new Range(range.getKey(), start, end));
-    }
-    List<Operator> ops = new ArrayList<Operator>();
-    for (HbaseMetastoreProto.PartitionKeyComparator.Operator op : proto.getOpList()) {
-      ops.add(new Operator(Operator.Type.valueOf(op.getType().name()), op.getKey(),
-          op.getVal()));
-    }
-    return new PartitionKeyComparator(proto.getNames(), proto.getTypes(), ranges, ops);
-  }
-
-  @Override
-  public byte[] toByteArray() {
-    HbaseMetastoreProto.PartitionKeyComparator.Builder builder = 
-        HbaseMetastoreProto.PartitionKeyComparator.newBuilder();
-    builder.setNames(names);
-    builder.setTypes(types);
-    for (int i=0;i<ranges.size();i++) {
-      Range range = ranges.get(i);
-      HbaseMetastoreProto.PartitionKeyComparator.Mark startMark = null;
-      if (range.start != null) {
-        startMark = HbaseMetastoreProto.PartitionKeyComparator.Mark.newBuilder()
-          .setValue(range.start.value)
-          .setInclusive(range.start.inclusive)
-          .build();
-      }
-      HbaseMetastoreProto.PartitionKeyComparator.Mark endMark = null;
-      if (range.end != null) {
-        endMark = HbaseMetastoreProto.PartitionKeyComparator.Mark.newBuilder()
-          .setValue(range.end.value)
-          .setInclusive(range.end.inclusive)
-          .build();
-      }
-        
-      HbaseMetastoreProto.PartitionKeyComparator.Range.Builder rangeBuilder = 
-        HbaseMetastoreProto.PartitionKeyComparator.Range.newBuilder();
-      rangeBuilder.setKey(range.keyName);
-      if (startMark != null) {
-        rangeBuilder.setStart(startMark);
-      }
-      if (endMark != null) {
-        rangeBuilder.setEnd(endMark);
-      }
-      builder.addRange(rangeBuilder.build());
-    }
-    for (int i=0;i<ops.size();i++) {
-      Operator op = ops.get(i);
-      builder.addOp(HbaseMetastoreProto.PartitionKeyComparator.Operator.newBuilder()
-        .setKey(op.keyName)
-        .setType(Type.valueOf(op.type.toString()))
-        .setVal(op.val).build());
-    }
-    return builder.build().toByteArray();
-  }
-
-  @Override
-  public int compareTo(byte[] value, int offset, int length) {
-    byte[] bytes = Arrays.copyOfRange(value, offset, offset + length);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Get key " + new String(bytes));
-    }
-    BinarySortableSerDe serDe = new BinarySortableSerDe();
-    List deserializedkeys = null;
-    try {
-      serDe.initialize(new Configuration(), serdeProps);
-      deserializedkeys = ((List)serDe.deserialize(new BytesWritable(bytes))).subList(2, 2 + names.split(",").length);
-    } catch (SerDeException e) {
-      // don't bother with failed deserialization, continue with next key
-      return 1;
-    }
-    for (int i=0;i<ranges.size();i++) {
-      Range range = ranges.get(i);
-      NativeRange nativeRange = nativeRanges.get(i);
-
-      Comparable partVal = (Comparable)deserializedkeys.get(nativeRange.pos);
-
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Try to match range " + partVal + ", start " + nativeRange.start + ", end "
-            + nativeRange.end);
-      }
-      if (range.start == null || range.start.inclusive && partVal.compareTo(nativeRange.start)>=0 ||
-          !range.start.inclusive && partVal.compareTo(nativeRange.start)>0) {
-        if (range.end == null || range.end.inclusive && partVal.compareTo(nativeRange.end)<=0 ||
-            !range.end.inclusive && partVal.compareTo(nativeRange.end)<0) {
-          continue;
-        }
-      }
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Fail to match range " + range.keyName + "-" + partVal + "[" + nativeRange.start
-            + "," + nativeRange.end + "]");
-      }
-      return 1;
-    }
-
-    for (int i=0;i<ops.size();i++) {
-      Operator op = ops.get(i);
-      NativeOperator nativeOp = nativeOps.get(i);
-      switch (op.type) {
-      case LIKE:
-        if (!deserializedkeys.get(nativeOp.pos).toString().matches(op.val)) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Fail to match operator " + op.keyName + "(" + deserializedkeys.get(nativeOp.pos)
-                + ") LIKE " + nativeOp.val);
-          }
-          return 1;
-        }
-        break;
-      case NOTEQUALS:
-        if (nativeOp.val.equals(deserializedkeys.get(nativeOp.pos))) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Fail to match operator " + op.keyName + "(" + deserializedkeys.get(nativeOp.pos)
-                + ")!=" + nativeOp.val);
-          }
-          return 1;
-        }
-        break;
-      }
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("All conditions satisfied:" + deserializedkeys);
-    }
-    return 0;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java
deleted file mode 100644
index de3b17b..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * A {@link org.apache.hadoop.hive.metastore.api.StorageDescriptor} with most of it's content
- * shallow copied from the underlying storage descriptor.  Location and parameters are left alone.
- * To avoid issues when users change the contents, all lists and nested structures (cols, serde,
- * buckets, sortCols, and skewed) are deep copied when they are accessed for reading or writing.
- * (It has to be done on read as well because there's no way to guarantee the user won't change the
- * nested structure or list, which would result in changing every storage descriptor sharing that
- * structure.)  Users wishing better performance can call setReadyOnly(), which will prevent the
- * copies.
- */
-public class SharedStorageDescriptor extends StorageDescriptor {
-  static final private Logger LOG = LoggerFactory.getLogger(SharedStorageDescriptor.class.getName());
-  private boolean colsCopied = false;
-  private boolean serdeCopied = false;
-  private boolean bucketsCopied = false;
-  private boolean sortCopied = false;
-  private boolean skewedCopied = false;
-
-  SharedStorageDescriptor() {
-  }
-
-  void setShared(StorageDescriptor shared) {
-    if (shared.getCols() != null) super.setCols(shared.getCols());
-    // Skip location
-    if (shared.getInputFormat() != null) super.setInputFormat(shared.getInputFormat());
-    if (shared.getOutputFormat() != null) super.setOutputFormat(shared.getOutputFormat());
-    super.setCompressed(shared.isCompressed());
-    super.setNumBuckets(shared.getNumBuckets());
-    if (shared.getSerdeInfo() != null) super.setSerdeInfo(shared.getSerdeInfo());
-    if (shared.getBucketCols() != null) super.setBucketCols(shared.getBucketCols());
-    if (shared.getSortCols() != null) super.setSortCols(shared.getSortCols());
-    // skip parameters
-    if (shared.getSkewedInfo() != null) super.setSkewedInfo(shared.getSkewedInfo());
-    super.setStoredAsSubDirectories(shared.isStoredAsSubDirectories());
-  }
-
-  /**
-   * Promise that you'll only use this shared storage descriptor in a read only mode.
-   * This prevents the copies of the nested structures and lists when reading them.  However, the
-   * caller must not change the structures or lists returned to it, as this will change all
-   * storage descriptor sharing that list.
-   */
-  public void setReadOnly() {
-    colsCopied = serdeCopied = bucketsCopied = sortCopied = skewedCopied = true;
-  }
-
-  @Override
-  public void addToCols(FieldSchema fs) {
-    copyCols();
-    super.addToCols(fs);
-  }
-
-  @Override
-  public List<FieldSchema> getCols() {
-    copyCols();
-    return super.getCols();
-  }
-
-  @Override
-  public void setCols(List<FieldSchema> cols) {
-    colsCopied = true;
-    super.setCols(cols);
-  }
-
-  @Override
-  public void unsetCols() {
-    colsCopied = true;
-    super.unsetCols();
-  }
-
-  @Override
-  public Iterator<FieldSchema> getColsIterator() {
-    copyCols();
-    return super.getColsIterator();
-  }
-
-  private void copyCols() {
-    if (!colsCopied) {
-      colsCopied = true;
-      if (super.getCols() != null) {
-        List<FieldSchema> cols = new ArrayList<FieldSchema>(super.getColsSize());
-        for (FieldSchema fs : super.getCols()) cols.add(new FieldSchema(fs));
-        super.setCols(cols);
-      }
-    }
-  }
-
-  @Override
-  public SerDeInfo getSerdeInfo() {
-    copySerde();
-    return super.getSerdeInfo();
-  }
-
-  @Override
-  public void setSerdeInfo(SerDeInfo serdeInfo) {
-    serdeCopied = true;
-    super.setSerdeInfo(serdeInfo);
-  }
-
-  @Override
-  public void unsetSerdeInfo() {
-    serdeCopied = true;
-    super.unsetSerdeInfo();
-  }
-
-  private void copySerde() {
-    if (!serdeCopied) {
-      serdeCopied = true;
-      if (super.getSerdeInfo() != null) super.setSerdeInfo(new SerDeInfo(super.getSerdeInfo()));
-    }
-  }
-
-  @Override
-  public void addToBucketCols(String bucket) {
-    copyBucketCols();
-    super.addToBucketCols(bucket);
-  }
-
-  @Override
-  public List<String> getBucketCols() {
-    copyBucketCols();
-    return super.getBucketCols();
-  }
-
-  @Override
-  public void setBucketCols(List<String> buckets) {
-    bucketsCopied = true;
-    super.setBucketCols(buckets);
-  }
-
-  @Override
-  public void unsetBucketCols() {
-    bucketsCopied = true;
-    super.unsetBucketCols();
-  }
-
-  @Override
-  public Iterator<String> getBucketColsIterator() {
-    copyBucketCols();
-    return super.getBucketColsIterator();
-  }
-
-  private void copyBucketCols() {
-    if (!bucketsCopied) {
-      bucketsCopied = true;
-      if (super.getBucketCols() != null) {
-        List<String> buckets = new ArrayList<String>(super.getBucketColsSize());
-        for (String bucket : super.getBucketCols()) buckets.add(bucket);
-        super.setBucketCols(buckets);
-      }
-    }
-  }
-
-  @Override
-  public void addToSortCols(Order sort) {
-    copySort();
-    super.addToSortCols(sort);
-  }
-
-  @Override
-  public List<Order> getSortCols() {
-    copySort();
-    return super.getSortCols();
-  }
-
-  @Override
-  public void setSortCols(List<Order> sorts) {
-    sortCopied = true;
-    super.setSortCols(sorts);
-  }
-
-  @Override
-  public void unsetSortCols() {
-    sortCopied = true;
-    super.unsetSortCols();
-  }
-
-  @Override
-  public Iterator<Order> getSortColsIterator() {
-    copySort();
-    return super.getSortColsIterator();
-  }
-
-  private void copySort() {
-    if (!sortCopied) {
-      sortCopied = true;
-      if (super.getSortCols() != null) {
-        List<Order> sortCols = new ArrayList<Order>(super.getSortColsSize());
-        for (Order sortCol : super.getSortCols()) sortCols.add(new Order(sortCol));
-        super.setSortCols(sortCols);
-      }
-    }
-  }
-
-  @Override
-  public SkewedInfo getSkewedInfo() {
-    copySkewed();
-    return super.getSkewedInfo();
-  }
-
-  @Override
-  public void setSkewedInfo(SkewedInfo skewedInfo) {
-    skewedCopied = true;
-    super.setSkewedInfo(skewedInfo);
-  }
-
-  @Override
-  public void unsetSkewedInfo() {
-    skewedCopied = true;
-    super.unsetSkewedInfo();
-  }
-
-  private void copySkewed() {
-    if (!skewedCopied) {
-      skewedCopied = true;
-      if (super.getSkewedInfo() != null) super.setSkewedInfo(new SkewedInfo(super.getSkewedInfo()));
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
deleted file mode 100644
index 78a962a..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.protobuf.ByteString;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.HiveStatsUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator;
-import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory;
-
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * A cache for stats.  This is only intended for use by
- * {@link org.apache.hadoop.hive.metastore.hbase.HBaseReadWrite} and should not be used outside
- * that class.
- */
-class StatsCache {
-
-  private static final Logger LOG = LoggerFactory.getLogger(StatsCache.class.getName());
-  private static StatsCache self = null;
-
-  private LoadingCache<StatsCacheKey, AggrStats> cache;
-  private Invalidator invalidator;
-  private long runInvalidatorEvery;
-  private long maxTimeInCache;
-  private boolean invalidatorHasRun;
-
-  @VisibleForTesting Counter misses;
-  @VisibleForTesting Counter hbaseHits;
-  @VisibleForTesting Counter totalGets;
-
-  static synchronized StatsCache getInstance(Configuration conf) {
-    if (self == null) {
-      self = new StatsCache(conf);
-    }
-    return self;
-  }
-
-  private StatsCache(final Configuration conf) {
-    final StatsCache me = this;
-    cache = CacheBuilder.newBuilder()
-        .maximumSize(
-            HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_CACHE_ENTRIES))
-        .expireAfterWrite(HiveConf.getTimeVar(conf,
-            HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_MEMORY_TTL, TimeUnit.SECONDS), TimeUnit.SECONDS)
-        .build(new CacheLoader<StatsCacheKey, AggrStats>() {
-          @Override
-          public AggrStats load(StatsCacheKey key) throws Exception {
-            boolean useDensityFunctionForNDVEstimation = HiveConf.getBoolVar(conf,
-                HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION);
-            double ndvTuner = HiveConf.getFloatVar(conf,
-                HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_TUNER);
-            HBaseReadWrite hrw = HBaseReadWrite.getInstance();
-            AggrStats aggrStats = hrw.getAggregatedStats(key.hashed);
-            if (aggrStats == null) {
-              misses.incr();
-              ColumnStatsAggregator aggregator = null;
-              aggrStats = new AggrStats();
-              LOG.debug("Unable to find aggregated stats for " + key.colName + ", aggregating");
-              List<ColumnStatistics> css = hrw.getPartitionStatistics(key.dbName, key.tableName,
-                  key.partNames, HBaseStore.partNameListToValsList(key.partNames),
-                  Collections.singletonList(key.colName));
-              if (css != null && css.size() > 0) {
-                aggrStats.setPartsFound(css.size());
-                if (aggregator == null) {
-                  aggregator = ColumnStatsAggregatorFactory.getColumnStatsAggregator(css.iterator()
-                      .next().getStatsObj().iterator().next().getStatsData().getSetField(),
-                      useDensityFunctionForNDVEstimation, ndvTuner);
-                }
-                ColumnStatisticsObj statsObj = aggregator
-                    .aggregate(key.colName, key.partNames, css);
-                aggrStats.addToColStats(statsObj);
-                me.put(key, aggrStats);
-              }
-            } else {
-              hbaseHits.incr();
-            }
-            return aggrStats;
-          }
-        });
-    misses = new Counter("Stats cache table misses");
-    hbaseHits = new Counter("Stats cache table hits");
-    totalGets = new Counter("Total get calls to the stats cache");
-
-    maxTimeInCache = HiveConf.getTimeVar(conf,
-        HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_HBASE_TTL, TimeUnit.SECONDS);
-    // We want runEvery in milliseconds, even though we give the default value in the conf in
-    // seconds.
-    runInvalidatorEvery = HiveConf.getTimeVar(conf,
-        HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_INVALIDATOR_FREQUENCY, TimeUnit.MILLISECONDS);
-
-    invalidator = new Invalidator();
-    invalidator.setDaemon(true);
-    invalidator.start();
-  }
-
-  /**
-   * Add an object to the cache.
-   * @param key Key for this entry
-   * @param aggrStats stats
-   * @throws java.io.IOException
-   */
-  void put(StatsCacheKey key, AggrStats aggrStats) throws IOException {
-    HBaseReadWrite.getInstance().putAggregatedStats(key.hashed, key.dbName, key.tableName,
-        key.partNames,
-        key.colName, aggrStats);
-    cache.put(key, aggrStats);
-  }
-
-  /**
-   * Get partition level statistics
-   * @param dbName name of database table is in
-   * @param tableName name of table
-   * @param partNames names of the partitions
-   * @param colName of column to get stats for
-   * @return stats object for this column, or null if none cached
-   * @throws java.io.IOException
-   */
-
-  AggrStats get(String dbName, String tableName, List<String> partNames, String colName)
-      throws IOException {
-    totalGets.incr();
-    StatsCacheKey key = new StatsCacheKey(dbName, tableName, partNames, colName);
-    try {
-      return cache.get(key);
-    } catch (ExecutionException e) {
-      throw new IOException(e);
-    }
-  }
-
-  /**
-   * Remove all entries that are related to a particular set of partitions.  This should be
-   * called when partitions are deleted or stats are updated.
-   * @param dbName name of database table is in
-   * @param tableName name of table
-   * @param partName name of the partition
-   * @throws IOException
-   */
-  void invalidate(String dbName, String tableName, String partName)
-      throws IOException {
-    invalidator.addToQueue(
-        HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry.newBuilder()
-            .setDbName(ByteString.copyFrom(dbName.getBytes(HBaseUtils.ENCODING)))
-            .setTableName(ByteString.copyFrom(tableName.getBytes(HBaseUtils.ENCODING)))
-            .setPartName(ByteString.copyFrom(partName.getBytes(HBaseUtils.ENCODING)))
-            .build());
-  }
-
-  void dumpCounters() {
-    LOG.debug(misses.dump());
-    LOG.debug(hbaseHits.dump());
-    LOG.debug(totalGets.dump());
-  }
-
-  /**
-   * Completely dump the cache from memory, used to test that we can access stats from HBase itself.
-   * @throws IOException
-   */
-  @VisibleForTesting void flushMemory() throws IOException {
-    cache.invalidateAll();
-  }
-
-  @VisibleForTesting void resetCounters() {
-    misses.clear();
-    hbaseHits.clear();
-    totalGets.clear();
-  }
-
-  @VisibleForTesting void setRunInvalidatorEvery(long runEvery) {
-    runInvalidatorEvery = runEvery;
-  }
-
-  @VisibleForTesting void setMaxTimeInCache(long maxTime) {
-    maxTimeInCache = maxTime;
-  }
-
-  @VisibleForTesting void wakeInvalidator() throws InterruptedException {
-    invalidatorHasRun = false;
-    // Wait through 2 cycles so we're sure our entry won't be picked as too new.
-    Thread.sleep(2 * runInvalidatorEvery);
-    invalidator.interrupt();
-    while (!invalidatorHasRun) {
-      Thread.sleep(10);
-    }
-  }
-
-  static class StatsCacheKey {
-    final byte[] hashed;
-    String dbName;
-    String tableName;
-    List<String> partNames;
-    String colName;
-    private MessageDigest md;
-
-    StatsCacheKey(byte[] key) {
-      hashed = key;
-    }
-
-    StatsCacheKey(String dbName, String tableName, List<String> partNames, String colName) {
-      this.dbName = dbName;
-      this.tableName = tableName;
-      this.partNames = partNames;
-      this.colName = colName;
-
-      try {
-        md = MessageDigest.getInstance("MD5");
-      } catch (NoSuchAlgorithmException e) {
-        throw new RuntimeException(e);
-      }
-      md.update(dbName.getBytes(HBaseUtils.ENCODING));
-      md.update(tableName.getBytes(HBaseUtils.ENCODING));
-      Collections.sort(this.partNames);
-      for (String s : partNames) {
-        md.update(s.getBytes(HBaseUtils.ENCODING));
-      }
-      md.update(colName.getBytes(HBaseUtils.ENCODING));
-      hashed = md.digest();
-    }
-
-    @Override
-    public boolean equals(Object other) {
-      if (other == null || !(other instanceof StatsCacheKey)) return false;
-      StatsCacheKey that = (StatsCacheKey)other;
-      return Arrays.equals(hashed, that.hashed);
-    }
-
-    @Override
-    public int hashCode() {
-      return Arrays.hashCode(hashed);
-    }
-  }
-
-  private class Invalidator extends Thread {
-    private List<HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry> entries = new ArrayList<>();
-    private Lock lock = new ReentrantLock();
-
-    void addToQueue(HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry entry) {
-      lock.lock();
-      try {
-        entries.add(entry);
-      } finally {
-        lock.unlock();
-      }
-    }
-
-    @Override
-    public void run() {
-      while (true) {
-        long startedAt = System.currentTimeMillis();
-        List<HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry> thisRun = null;
-        lock.lock();
-        try {
-          if (entries.size() > 0) {
-            thisRun = entries;
-            entries = new ArrayList<>();
-          }
-        } finally {
-          lock.unlock();
-        }
-
-        if (thisRun != null) {
-          try {
-            HbaseMetastoreProto.AggrStatsInvalidatorFilter filter =
-                HbaseMetastoreProto.AggrStatsInvalidatorFilter.newBuilder()
-                .setRunEvery(runInvalidatorEvery)
-                .setMaxCacheEntryLife(maxTimeInCache)
-                .addAllToInvalidate(thisRun)
-                .build();
-            List<StatsCacheKey> keys =
-                HBaseReadWrite.getInstance().invalidateAggregatedStats(filter);
-            cache.invalidateAll(keys);
-          } catch (IOException e) {
-            // Not a lot I can do here
-            LOG.error("Caught error while invalidating entries in the cache", e);
-          }
-        }
-        invalidatorHasRun = true;
-
-        try {
-          sleep(runInvalidatorEvery - (System.currentTimeMillis() - startedAt));
-        } catch (InterruptedException e) {
-          LOG.warn("Interupted while sleeping", e);
-        }
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/TephraHBaseConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/TephraHBaseConnection.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/TephraHBaseConnection.java
deleted file mode 100644
index f66200f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/TephraHBaseConnection.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import co.cask.tephra.TransactionAware;
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.distributed.ThreadLocalClientProvider;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import co.cask.tephra.hbase10.TransactionAwareHTable;
-import co.cask.tephra.hbase10.coprocessor.TransactionProcessor;
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.twill.discovery.InMemoryDiscoveryService;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A class that uses Tephra for transaction management.
- */
-public class TephraHBaseConnection extends VanillaHBaseConnection {
-  static final private Logger LOG = LoggerFactory.getLogger(TephraHBaseConnection.class.getName());
-
-  private Map<String, TransactionAware> txnTables;
-  private TransactionContext txn;
-  private TransactionSystemClient txnClient;
-
-  TephraHBaseConnection() {
-    super();
-    txnTables = new HashMap<String, TransactionAware>();
-  }
-
-  @Override
-  public void connect() throws IOException {
-    super.connect();
-    if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEST)) {
-      LOG.debug("Using an in memory client transaction system for testing");
-      TransactionManager txnMgr = new TransactionManager(conf);
-      txnMgr.startAndWait();
-      txnClient = new InMemoryTxSystemClient(txnMgr);
-    } else {
-      // TODO should enable use of ZKDiscoveryService if users want it
-      LOG.debug("Using real client transaction system for production");
-      txnClient = new TransactionServiceClient(conf,
-          new ThreadLocalClientProvider(conf, new InMemoryDiscoveryService()));
-    }
-    for (String tableName : HBaseReadWrite.tableNames) {
-      txnTables.put(tableName, new TransactionAwareHTable(super.getHBaseTable(tableName, true)));
-    }
-    txn = new TransactionContext(txnClient, txnTables.values());
-  }
-
-  @Override
-  public void beginTransaction() throws IOException {
-    try {
-      txn.start();
-      LOG.debug("Started txn in tephra");
-    } catch (TransactionFailureException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void commitTransaction() throws IOException {
-    try {
-      txn.finish();
-      LOG.debug("Finished txn in tephra");
-    } catch (TransactionFailureException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void rollbackTransaction() throws IOException {
-    try {
-      txn.abort();
-      LOG.debug("Aborted txn in tephra");
-    } catch (TransactionFailureException e) {
-      throw new IOException(e);
-    }
-  }
-
-  @Override
-  public void flush(HTableInterface htab) throws IOException {
-    // NO-OP as we want to flush at commit time
-  }
-
-  @Override
-  protected HTableDescriptor buildDescriptor(String tableName, List<byte[]> columnFamilies)
-      throws IOException {
-    HTableDescriptor tableDesc = super.buildDescriptor(tableName, columnFamilies);
-    tableDesc.addCoprocessor(TransactionProcessor.class.getName());
-    return tableDesc;
-  }
-
-  @Override
-  public HTableInterface getHBaseTable(String tableName, boolean force) throws IOException {
-    // Ignore force, it will mess up our previous creation of the tables.
-    return (TransactionAwareHTable)txnTables.get(tableName);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/VanillaHBaseConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/VanillaHBaseConnection.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/VanillaHBaseConnection.java
deleted file mode 100644
index e631580..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/VanillaHBaseConnection.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A pass through to a simple HBase connection.  This has no transactions.
- */
-public class VanillaHBaseConnection implements HBaseConnection {
-  static final private Logger LOG = LoggerFactory.getLogger(VanillaHBaseConnection.class.getName());
-
-  protected HConnection conn;
-  protected Map<String, HTableInterface> tables;
-  protected Configuration conf;
-
-  VanillaHBaseConnection() {
-    tables = new HashMap<String, HTableInterface>();
-  }
-
-  @Override
-  public void connect() throws IOException {
-    if (conf == null) throw new RuntimeException("Must call getConf before connect");
-    conn = HConnectionManager.createConnection(conf);
-  }
-
-  @Override
-  public void close() throws IOException {
-    for (HTableInterface htab : tables.values()) htab.close();
-  }
-
-  @Override
-  public void beginTransaction() throws IOException {
-
-  }
-
-  @Override
-  public void commitTransaction() throws IOException {
-
-  }
-
-  @Override
-  public void rollbackTransaction() throws IOException {
-
-  }
-
-  @Override
-  public void flush(HTableInterface htab) throws IOException {
-    htab.flushCommits();
-  }
-
-  @Override
-  public void createHBaseTable(String tableName, List<byte[]> columnFamilies)
-      throws IOException {
-    HBaseAdmin admin = new HBaseAdmin(conn);
-    LOG.info("Creating HBase table " + tableName);
-    admin.createTable(buildDescriptor(tableName, columnFamilies));
-    admin.close();
-  }
-
-  protected HTableDescriptor buildDescriptor(String tableName, List<byte[]> columnFamilies)
-      throws IOException {
-    HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
-    for (byte[] cf : columnFamilies) {
-      tableDesc.addFamily(new HColumnDescriptor(cf));
-    }
-    return tableDesc;
-  }
-
-  @Override
-  public HTableInterface getHBaseTable(String tableName) throws IOException {
-    return getHBaseTable(tableName, false);
-  }
-
-  @Override
-  public HTableInterface getHBaseTable(String tableName, boolean force) throws IOException {
-    HTableInterface htab = tables.get(tableName);
-    if (htab == null) {
-      LOG.debug("Trying to connect to table " + tableName);
-      try {
-        htab = conn.getTable(tableName);
-        // Calling gettable doesn't actually connect to the region server, it's very light
-        // weight, so call something else so we actually reach out and touch the region server
-        // and see if the table is there.
-        if (force) htab.get(new Get("nosuchkey".getBytes(HBaseUtils.ENCODING)));
-      } catch (IOException e) {
-        LOG.info("Caught exception when table was missing");
-        return null;
-      }
-      htab.setAutoFlushTo(false);
-      tables.put(tableName, htab);
-    }
-    return htab;
-  }
-
-  @Override
-  public void setConf(Configuration conf) {
-    this.conf = conf;
-  }
-
-  @Override
-  public Configuration getConf() {
-    return conf;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
deleted file mode 100644
index e6c836b..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-public class BinaryColumnStatsAggregator extends ColumnStatsAggregator {
-
-  @Override
-  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
-      List<ColumnStatistics> css) throws MetaException {
-    ColumnStatisticsObj statsObj = null;
-    BinaryColumnStatsData aggregateData = null;
-    String colType = null;
-    for (ColumnStatistics cs : css) {
-      if (cs.getStatsObjSize() != 1) {
-        throw new MetaException(
-            "The number of columns should be exactly one in aggrStats, but found "
-                + cs.getStatsObjSize());
-      }
-      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-      if (statsObj == null) {
-        colType = cso.getColType();
-        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
-            .getStatsData().getSetField());
-      }
-      BinaryColumnStatsData newData = cso.getStatsData().getBinaryStats();
-      if (aggregateData == null) {
-        aggregateData = newData.deepCopy();
-      } else {
-        aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
-        aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
-        aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-      }
-    }
-    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
-    columnStatisticsData.setBinaryStats(aggregateData);
-    statsObj.setStatsData(columnStatisticsData);
-    return statsObj;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
deleted file mode 100644
index a34bc9f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-public class BooleanColumnStatsAggregator extends ColumnStatsAggregator {
-
-  @Override
-  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
-      List<ColumnStatistics> css) throws MetaException {
-    ColumnStatisticsObj statsObj = null;
-    BooleanColumnStatsData aggregateData = null;
-    String colType = null;
-    for (ColumnStatistics cs : css) {
-      if (cs.getStatsObjSize() != 1) {
-        throw new MetaException(
-            "The number of columns should be exactly one in aggrStats, but found "
-                + cs.getStatsObjSize());
-      }
-      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-      if (statsObj == null) {
-        colType = cso.getColType();
-        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
-            .getStatsData().getSetField());
-      }
-      BooleanColumnStatsData newData = cso.getStatsData().getBooleanStats();
-      if (aggregateData == null) {
-        aggregateData = newData.deepCopy();
-      } else {
-        aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
-        aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
-        aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-      }
-    }
-    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
-    columnStatisticsData.setBooleanStats(aggregateData);
-    statsObj.setStatsData(columnStatisticsData);
-    return statsObj;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
deleted file mode 100644
index a52e5e5..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-public abstract class ColumnStatsAggregator {
-  public boolean useDensityFunctionForNDVEstimation;
-  public double ndvTuner;
-  public abstract ColumnStatisticsObj aggregate(String colName, List<String> partNames,
-      List<ColumnStatistics> css) throws MetaException;
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
deleted file mode 100644
index 173e06f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-
-public class ColumnStatsAggregatorFactory {
-
-  private ColumnStatsAggregatorFactory() {
-  }
-
-  public static ColumnStatsAggregator getColumnStatsAggregator(_Fields type,
-      boolean useDensityFunctionForNDVEstimation, double ndvTuner) {
-    ColumnStatsAggregator agg;
-    switch (type) {
-    case BOOLEAN_STATS:
-      agg = new BooleanColumnStatsAggregator();
-      break;
-    case LONG_STATS:
-      agg = new LongColumnStatsAggregator();
-      break;
-    case DATE_STATS:
-      agg = new DateColumnStatsAggregator();
-      break;
-    case DOUBLE_STATS:
-      agg = new DoubleColumnStatsAggregator();
-      break;
-    case STRING_STATS:
-      agg = new StringColumnStatsAggregator();
-      break;
-    case BINARY_STATS:
-      agg = new BinaryColumnStatsAggregator();
-      break;
-    case DECIMAL_STATS:
-      agg = new DecimalColumnStatsAggregator();
-      break;
-    default:
-      throw new RuntimeException("Woh, bad.  Unknown stats type " + type.toString());
-    }
-    agg.useDensityFunctionForNDVEstimation = useDensityFunctionForNDVEstimation;
-    agg.ndvTuner = ndvTuner;
-    return agg;
-  }
-
-  public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
-    ColumnStatisticsObj cso = new ColumnStatisticsObj();
-    ColumnStatisticsData csd = new ColumnStatisticsData();
-    cso.setColName(colName);
-    cso.setColType(colType);
-    switch (type) {
-    case BOOLEAN_STATS:
-      csd.setBooleanStats(new BooleanColumnStatsData());
-      break;
-
-    case LONG_STATS:
-      csd.setLongStats(new LongColumnStatsData());
-      break;
-
-    case DATE_STATS:
-      csd.setDateStats(new DateColumnStatsData());
-      break;
-
-    case DOUBLE_STATS:
-      csd.setDoubleStats(new DoubleColumnStatsData());
-      break;
-
-    case STRING_STATS:
-      csd.setStringStats(new StringColumnStatsData());
-      break;
-
-    case BINARY_STATS:
-      csd.setBinaryStats(new BinaryColumnStatsData());
-      break;
-
-    case DECIMAL_STATS:
-      csd.setDecimalStats(new DecimalColumnStatsData());
-      break;
-
-    default:
-      throw new RuntimeException("Woh, bad.  Unknown stats type!");
-    }
-
-    cso.setStatsData(csd);
-    return cso;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
deleted file mode 100644
index 5924c3e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
+++ /dev/null
@@ -1,371 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.StatObjectConverter;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.hbase.HBaseUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implements
-    IExtrapolatePartStatus {
-  
-  private static final Logger LOG = LoggerFactory.getLogger(DecimalColumnStatsAggregator.class);
-
-  @Override
-  public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
-      List<ColumnStatistics> css) throws MetaException {
-    ColumnStatisticsObj statsObj = null;
-
-    // check if all the ColumnStatisticsObjs contain stats and all the ndv are
-    // bitvectors
-    boolean doAllPartitionContainStats = partNames.size() == css.size();
-    LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
-    NumDistinctValueEstimator ndvEstimator = null;
-    String colType = null;
-    for (ColumnStatistics cs : css) {
-      if (cs.getStatsObjSize() != 1) {
-        throw new MetaException(
-            "The number of columns should be exactly one in aggrStats, but found "
-                + cs.getStatsObjSize());
-      }
-      ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-      if (statsObj == null) {
-        colType = cso.getColType();
-        statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
-            .getStatsData().getSetField());
-      }
-      if (!cso.getStatsData().getDecimalStats().isSetBitVectors()
-          || cso.getStatsData().getDecimalStats().getBitVectors().length() == 0) {
-        ndvEstimator = null;
-        break;
-      } else {
-        // check if all of the bit vectors can merge
-        NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
-            .getNumDistinctValueEstimator(cso.getStatsData().getDecimalStats().getBitVectors());
-        if (ndvEstimator == null) {
-          ndvEstimator = estimator;
-        } else {
-          if (ndvEstimator.canMerge(estimator)) {
-            continue;
-          } else {
-            ndvEstimator = null;
-            break;
-          }
-        }
-      }
-    }
-    if (ndvEstimator != null) {
-      ndvEstimator = NumDistinctValueEstimatorFactory
-          .getEmptyNumDistinctValueEstimator(ndvEstimator);
-    }
-    LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
-    ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
-    if (doAllPartitionContainStats || css.size() < 2) {
-      DecimalColumnStatsData aggregateData = null;
-      long lowerBound = 0;
-      long higherBound = 0;
-      double densityAvgSum = 0.0;
-      for (ColumnStatistics cs : css) {
-        ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-        DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
-        lowerBound = Math.max(lowerBound, newData.getNumDVs());
-        higherBound += newData.getNumDVs();
-        densityAvgSum += (HBaseUtils.getDoubleValue(newData.getHighValue()) - HBaseUtils
-            .getDoubleValue(newData.getLowValue())) / newData.getNumDVs();
-        if (ndvEstimator != null) {
-          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
-              .getNumDistinctValueEstimator(newData.getBitVectors()));
-        }
-        if (aggregateData == null) {
-          aggregateData = newData.deepCopy();
-        } else {
-          if (HBaseUtils.getDoubleValue(aggregateData.getLowValue()) < HBaseUtils
-              .getDoubleValue(newData.getLowValue())) {
-            aggregateData.setLowValue(aggregateData.getLowValue());
-          } else {
-            aggregateData.setLowValue(newData.getLowValue());
-          }
-          if (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) > HBaseUtils
-              .getDoubleValue(newData.getHighValue())) {
-            aggregateData.setHighValue(aggregateData.getHighValue());
-          } else {
-            aggregateData.setHighValue(newData.getHighValue());
-          }
-          aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-          aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
-        }
-      }
-      if (ndvEstimator != null) {
-        // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
-        // use uniform distribution assumption because we can merge bitvectors
-        // to get a good estimation.
-        aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
-      } else {
-        long estimation;
-        if (useDensityFunctionForNDVEstimation) {
-          // We have estimation, lowerbound and higherbound. We use estimation
-          // if it is between lowerbound and higherbound.
-          double densityAvg = densityAvgSum / partNames.size();
-          estimation = (long) ((HBaseUtils.getDoubleValue(aggregateData.getHighValue()) - HBaseUtils
-              .getDoubleValue(aggregateData.getLowValue())) / densityAvg);
-          if (estimation < lowerBound) {
-            estimation = lowerBound;
-          } else if (estimation > higherBound) {
-            estimation = higherBound;
-          }
-        } else {
-          estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
-        }
-        aggregateData.setNumDVs(estimation);
-      }
-      columnStatisticsData.setDecimalStats(aggregateData);
-    } else {
-      // we need extrapolation
-      LOG.debug("start extrapolation for " + colName);
-      Map<String, Integer> indexMap = new HashMap<String, Integer>();
-      for (int index = 0; index < partNames.size(); index++) {
-        indexMap.put(partNames.get(index), index);
-      }
-      Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
-      Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
-      // while we scan the css, we also get the densityAvg, lowerbound and
-      // higerbound when useDensityFunctionForNDVEstimation is true.
-      double densityAvgSum = 0.0;
-      if (ndvEstimator == null) {
-        // if not every partition uses bitvector for ndv, we just fall back to
-        // the traditional extrapolation methods.
-        for (ColumnStatistics cs : css) {
-          String partName = cs.getStatsDesc().getPartName();
-          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-          DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
-          if (useDensityFunctionForNDVEstimation) {
-            densityAvgSum += (HBaseUtils.getDoubleValue(newData.getHighValue()) - HBaseUtils
-                .getDoubleValue(newData.getLowValue())) / newData.getNumDVs();
-          }
-          adjustedIndexMap.put(partName, (double) indexMap.get(partName));
-          adjustedStatsMap.put(partName, cso.getStatsData());
-        }
-      } else {
-        // we first merge all the adjacent bitvectors that we could merge and
-        // derive new partition names and index.
-        StringBuilder pseudoPartName = new StringBuilder();
-        double pseudoIndexSum = 0;
-        int length = 0;
-        int curIndex = -1;
-        DecimalColumnStatsData aggregateData = null;
-        for (ColumnStatistics cs : css) {
-          String partName = cs.getStatsDesc().getPartName();
-          ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
-          DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
-          // newData.isSetBitVectors() should be true for sure because we
-          // already checked it before.
-          if (indexMap.get(partName) != curIndex) {
-            // There is bitvector, but it is not adjacent to the previous ones.
-            if (length > 0) {
-              // we have to set ndv
-              adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
-              aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
-              ColumnStatisticsData csd = new ColumnStatisticsData();
-              csd.setDecimalStats(aggregateData);
-              adjustedStatsMap.put(pseudoPartName.toString(), csd);
-              if (useDensityFunctionForNDVEstimation) {
-                densityAvgSum += (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) - HBaseUtils
-                    .getDoubleValue(aggregateData.getLowValue())) / aggregateData.getNumDVs();
-              }
-              // reset everything
-              pseudoPartName = new StringBuilder();
-              pseudoIndexSum = 0;
-              length = 0;
-              ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
-            }
-            aggregateData = null;
-          }
-          curIndex = indexMap.get(partName);
-          pseudoPartName.append(partName);
-          pseudoIndexSum += curIndex;
-          length++;
-          curIndex++;
-          if (aggregateData == null) {
-            aggregateData = newData.deepCopy();
-          } else {
-            if (HBaseUtils.getDoubleValue(aggregateData.getLowValue()) < HBaseUtils
-                .getDoubleValue(newData.getLowValue())) {
-              aggregateData.setLowValue(aggregateData.getLowValue());
-            } else {
-              aggregateData.setLowValue(newData.getLowValue());
-            }
-            if (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) > HBaseUtils
-                .getDoubleValue(newData.getHighValue())) {
-              aggregateData.setHighValue(aggregateData.getHighValue());
-            } else {
-              aggregateData.setHighValue(newData.getHighValue());
-            }
-            aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-          }
-          ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
-              .getNumDistinctValueEstimator(newData.getBitVectors()));
-        }
-        if (length > 0) {
-          // we have to set ndv
-          adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
-          aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
-          ColumnStatisticsData csd = new ColumnStatisticsData();
-          csd.setDecimalStats(aggregateData);
-          adjustedStatsMap.put(pseudoPartName.toString(), csd);
-          if (useDensityFunctionForNDVEstimation) {
-            densityAvgSum += (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) - HBaseUtils
-                .getDoubleValue(aggregateData.getLowValue())) / aggregateData.getNumDVs();
-          }
-        }
-      }
-      extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
-          adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
-    }
-    statsObj.setStatsData(columnStatisticsData);
-    LOG.debug("Ndv estimatation for " + colName + " is "
-        + columnStatisticsData.getDecimalStats().getNumDVs());
-    return statsObj;
-  }
-
-  @Override
-  public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
-      int numPartsWithStats, Map<String, Double> adjustedIndexMap,
-      Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
-    int rightBorderInd = numParts;
-    DecimalColumnStatsData extrapolateDecimalData = new DecimalColumnStatsData();
-    Map<String, DecimalColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
-    for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
-      extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDecimalStats());
-    }
-    List<Map.Entry<String, DecimalColumnStatsData>> list = new LinkedList<Map.Entry<String, DecimalColumnStatsData>>(
-        extractedAdjustedStatsMap.entrySet());
-    // get the lowValue
-    Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
-      public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
-          Map.Entry<String, DecimalColumnStatsData> o2) {
-        return o1.getValue().getLowValue().compareTo(o2.getValue().getLowValue());
-      }
-    });
-    double minInd = adjustedIndexMap.get(list.get(0).getKey());
-    double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
-    double lowValue = 0;
-    double min = HBaseUtils.getDoubleValue(list.get(0).getValue().getLowValue());
-    double max = HBaseUtils.getDoubleValue(list.get(list.size() - 1).getValue().getLowValue());
-    if (minInd == maxInd) {
-      lowValue = min;
-    } else if (minInd < maxInd) {
-      // left border is the min
-      lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
-    } else {
-      // right border is the min
-      lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
-    }
-
-    // get the highValue
-    Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
-      public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
-          Map.Entry<String, DecimalColumnStatsData> o2) {
-        return o1.getValue().getHighValue().compareTo(o2.getValue().getHighValue());
-      }
-    });
-    minInd = adjustedIndexMap.get(list.get(0).getKey());
-    maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
-    double highValue = 0;
-    min = HBaseUtils.getDoubleValue(list.get(0).getValue().getHighValue());
-    max = HBaseUtils.getDoubleValue(list.get(list.size() - 1).getValue().getHighValue());
-    if (minInd == maxInd) {
-      highValue = min;
-    } else if (minInd < maxInd) {
-      // right border is the max
-      highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
-    } else {
-      // left border is the max
-      highValue = (min + (max - min) * minInd / (minInd - maxInd));
-    }
-
-    // get the #nulls
-    long numNulls = 0;
-    for (Map.Entry<String, DecimalColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
-      numNulls += entry.getValue().getNumNulls();
-    }
-    // we scale up sumNulls based on the number of partitions
-    numNulls = numNulls * numParts / numPartsWithStats;
-
-    // get the ndv
-    long ndv = 0;
-    long ndvMin = 0;
-    long ndvMax = 0;
-    Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
-      public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
-          Map.Entry<String, DecimalColumnStatsData> o2) {
-        return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
-      }
-    });
-    long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
-    long higherBound = 0;
-    for (Map.Entry<String, DecimalColumnStatsData> entry : list) {
-      higherBound += entry.getValue().getNumDVs();
-    }
-    if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
-      ndv = (long) ((highValue - lowValue) / densityAvg);
-      if (ndv < lowerBound) {
-        ndv = lowerBound;
-      } else if (ndv > higherBound) {
-        ndv = higherBound;
-      }
-    } else {
-      minInd = adjustedIndexMap.get(list.get(0).getKey());
-      maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
-      ndvMin = list.get(0).getValue().getNumDVs();
-      ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
-      if (minInd == maxInd) {
-        ndv = ndvMin;
-      } else if (minInd < maxInd) {
-        // right border is the max
-        ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
-      } else {
-        // left border is the max
-        ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
-      }
-    }
-    extrapolateDecimalData.setLowValue(StatObjectConverter.createThriftDecimal(String
-        .valueOf(lowValue)));
-    extrapolateDecimalData.setHighValue(StatObjectConverter.createThriftDecimal(String
-        .valueOf(highValue)));
-    extrapolateDecimalData.setNumNulls(numNulls);
-    extrapolateDecimalData.setNumDVs(ndv);
-    extrapolateData.setDecimalStats(extrapolateDecimalData);
-  }
-}


[09/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
deleted file mode 100644
index b4f8734..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.conf.Configuration;
-
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * A tool to dump contents from the HBase store in a human readable form
- */
-public class HBaseSchemaTool {
-
-  public static void main(String[] args) {
-    Options options = new Options();
-
-    options.addOption(OptionBuilder
-        .withLongOpt("help")
-        .withDescription("You're looking at it")
-        .create('h'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("install")
-        .withDescription("Install the schema onto an HBase cluster.")
-        .create('i'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("key")
-        .withDescription("Key to scan with.  This should be an exact key (not a regular expression")
-        .hasArg()
-        .create('k'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("list-tables")
-        .withDescription("List tables in HBase metastore")
-        .create('l'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("regex-key")
-        .withDescription("Regular expression to scan keys with.")
-        .hasArg()
-        .create('r'));
-
-    options.addOption(OptionBuilder
-        .withLongOpt("table")
-        .withDescription("HBase metastore table to scan")
-        .hasArg()
-        .create('t'));
-
-    CommandLine cli = null;
-    try {
-      cli = new GnuParser().parse(options, args);
-    } catch (ParseException e) {
-      System.err.println("Parse Exception: " + e.getMessage());
-      usage(options);
-      return;
-    }
-
-    if (cli.hasOption('h')) {
-      usage(options);
-      return;
-    }
-
-    Configuration conf = new Configuration();
-
-    if (cli.hasOption('i')) {
-      new HBaseSchemaTool().install(conf, System.err);
-      return;
-    }
-
-    String key = null;
-    if (cli.hasOption('k')) key = cli.getOptionValue('k');
-    String regex = null;
-    if (cli.hasOption('r')) regex = cli.getOptionValue('r');
-    if (key != null && regex != null) {
-      usage(options);
-      return;
-    }
-    if (key == null && regex == null) regex = ".*";
-
-    // I do this in the object rather than in the static main so that it's easier to test.
-    new HBaseSchemaTool().go(cli.hasOption('l'), cli.getOptionValue('t'), key, regex, conf,
-        System.out, System.err);
-  }
-
-  private static void usage(Options options) {
-    HelpFormatter formatter = new HelpFormatter();
-    String header = "This tool dumps contents of your hbase metastore.  You need to specify\n" +
-        "the table to dump.  You can optionally specify a regular expression on the key for\n" +
-        "the table.  Keep in mind that the key is often a compound.  For partitions regular\n" +
-        "expressions are not used because non-string values are\nstored in binary.  Instead for " +
-        "partition you can specify as much of the exact prefix as you want.  So you can give " +
-        "dbname.tablename or dbname.tablename.pval1...";
-    String footer = "If neither key or regex is provided a regex of .* will be assumed.  You\n" +
-        "cannot set both key and regex.";
-    formatter.printHelp("hbaseschematool", header, options, footer);
-    return;
-  }
-
-  @VisibleForTesting void go(boolean listTables, String table, String key, String regex,
-                             Configuration conf, PrintStream out, PrintStream err) {
-    List<String> lines = new ArrayList<>();
-    if (listTables) {
-      lines = Arrays.asList(HBaseReadWrite.tableNames);
-    } else {
-      // If they've used '.' as a key separator we need to replace it with the separator used by
-      // HBaseUtils
-      if (key != null) key = key.replace('.', HBaseUtils.KEY_SEPARATOR);
-      try {
-        HBaseReadWrite.setConf(conf);
-        HBaseReadWrite hrw = HBaseReadWrite.getInstance();
-        if (table.equalsIgnoreCase(HBaseReadWrite.DB_TABLE)) {
-          if (key != null) lines.add(hrw.printDatabase(key));
-          else lines.addAll(hrw.printDatabases(regex));
-        } else if (table.equalsIgnoreCase(HBaseReadWrite.FUNC_TABLE)) {
-          if (key != null) lines.add(hrw.printFunction(key));
-          else lines.addAll(hrw.printFunctions(regex));
-        } else if (table.equalsIgnoreCase(HBaseReadWrite.GLOBAL_PRIVS_TABLE)) {
-          // Ignore whatever they passed, there's always only either one or zero global privileges
-          lines.add(hrw.printGlobalPrivs());
-        } else if (table.equalsIgnoreCase(HBaseReadWrite.PART_TABLE)) {
-          if (key != null) lines.add(hrw.printPartition(key));
-          else lines.addAll(hrw.printPartitions(regex));
-        } else if (table.equalsIgnoreCase(HBaseReadWrite.USER_TO_ROLE_TABLE)) {
-          if (key != null) lines.add(hrw.printRolesForUser(key));
-          else lines.addAll(hrw.printRolesForUsers(regex));
-        } else if (table.equalsIgnoreCase(HBaseReadWrite.ROLE_TABLE)) {
-          if (key != null) lines.add(hrw.printRole(key));
-          else lines.addAll(hrw.printRoles(regex));
-        } else if (table.equalsIgnoreCase(HBaseReadWrite.TABLE_TABLE)) {
-          if (key != null) lines.add(hrw.printTable(key));
-          else lines.addAll(hrw.printTables(regex));
-        } else if (table.equalsIgnoreCase(HBaseReadWrite.SD_TABLE)) {
-          if (key != null) lines.add(hrw.printStorageDescriptor(Base64.decodeBase64(key)));
-          else lines.addAll(hrw.printStorageDescriptors());
-        } else if (table.equalsIgnoreCase(HBaseReadWrite.SECURITY_TABLE)) {
-          // We always print all of security, we don't worry about finding particular entries.
-          lines.addAll(hrw.printSecurity());
-        } else if (table.equalsIgnoreCase(HBaseReadWrite.SEQUENCES_TABLE)) {
-          // We always print all of sequences, we don't worry about finding particular entries.
-          lines.addAll(hrw.printSequences());
-        } else {
-          err.println("Unknown table: " + table);
-          return;
-        }
-      } catch (Exception e) {
-        err.println("Caught exception " + e.getClass() + " with message: " + e.getMessage());
-        return;
-      }
-    }
-    for (String line : lines) out.println(line);
-  }
-
-  @VisibleForTesting void install(Configuration conf, PrintStream err) {
-    try {
-      // We need to set the conf because createTablesIfNotExist will get a thread local version
-      // which requires that the configuration object be set.
-      HBaseReadWrite.setConf(conf);
-      HBaseReadWrite.createTablesIfNotExist();
-    } catch (Exception e) {
-      err.println("Caught exception " + e.getClass() + " with message: " + e.getMessage());
-      return;
-    }
-  }
-}


[17/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5e061557
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5e061557
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5e061557

Branch: refs/heads/master
Commit: 5e0615573c1fecf3ece7ff238b76bbce45a0337c
Parents: c9c7162
Author: Alan Gates <ga...@hortonworks.com>
Authored: Fri Aug 4 14:08:56 2017 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Fri Aug 4 14:08:56 2017 -0700

----------------------------------------------------------------------
 bin/ext/hbaseimport.sh                          |    27 -
 bin/ext/hbaseschematool.sh                      |    27 -
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    52 -
 .../metastore/hbase/HBaseIntegrationTests.java  |   112 -
 .../TestHBaseAggrStatsCacheIntegration.java     |   691 -
 .../hive/metastore/hbase/TestHBaseImport.java   |   775 -
 .../hbase/TestHBaseMetastoreMetrics.java        |   130 -
 .../metastore/hbase/TestHBaseMetastoreSql.java  |   223 -
 .../metastore/hbase/TestHBaseSchemaTool.java    |   582 -
 .../metastore/hbase/TestHBaseSchemaTool2.java   |    59 -
 .../hbase/TestHBaseStoreIntegration.java        |  1801 -
 .../hbase/TestStorageDescriptorSharing.java     |   191 -
 .../hadoop/hive/ql/TestLocationQueries.java     |     2 +-
 .../hadoop/hive/accumulo/AccumuloQTestUtil.java |     2 +-
 .../hive/cli/control/AbstractCliConfig.java     |     4 +-
 .../control/AbstractCoreBlobstoreCliDriver.java |     3 +-
 .../hadoop/hive/cli/control/CoreCliDriver.java  |     4 +-
 .../hive/cli/control/CoreCompareCliDriver.java  |     2 +-
 .../hive/cli/control/CoreNegativeCliDriver.java |     2 +-
 .../hive/cli/control/CorePerfCliDriver.java     |     2 +-
 .../hadoop/hive/hbase/HBaseQTestUtil.java       |     2 +-
 .../metastore/hbase/HBaseStoreTestUtil.java     |    45 -
 .../org/apache/hadoop/hive/ql/QTestUtil.java    |    54 +-
 .../hadoop/hive/ql/parse/CoreParseNegative.java |     3 +-
 metastore/pom.xml                               |     5 -
 .../metastore/hbase/HbaseMetastoreProto.java    | 46709 -----------------
 .../hadoop/hive/metastore/MetaStoreUtils.java   |    97 +
 .../hive/metastore/cache/SharedCache.java       |     6 +-
 .../aggr/BinaryColumnStatsAggregator.java       |    64 +
 .../aggr/BooleanColumnStatsAggregator.java      |    65 +
 .../columnstats/aggr/ColumnStatsAggregator.java |    33 +
 .../aggr/ColumnStatsAggregatorFactory.java      |   113 +
 .../aggr/DecimalColumnStatsAggregator.java      |   371 +
 .../aggr/DoubleColumnStatsAggregator.java       |   345 +
 .../aggr/IExtrapolatePartStatus.java            |    47 +
 .../aggr/LongColumnStatsAggregator.java         |   344 +
 .../merge/BinaryColumnStatsMerger.java          |    35 +
 .../merge/BooleanColumnStatsMerger.java         |    35 +
 .../columnstats/merge/ColumnStatsMerger.java    |    31 +
 .../merge/ColumnStatsMergerFactory.java         |   123 +
 .../merge/DateColumnStatsMerger.java            |    61 +
 .../merge/DecimalColumnStatsMerger.java         |    63 +
 .../merge/DoubleColumnStatsMerger.java          |    56 +
 .../merge/LongColumnStatsMerger.java            |    56 +
 .../merge/StringColumnStatsMerger.java          |    56 +
 .../hbase/AggrStatsInvalidatorFilter.java       |   120 -
 .../hadoop/hive/metastore/hbase/Counter.java    |    53 -
 .../hive/metastore/hbase/HBaseConnection.java   |    96 -
 .../metastore/hbase/HBaseFilterPlanUtil.java    |   612 -
 .../hive/metastore/hbase/HBaseImport.java       |   619 -
 .../hive/metastore/hbase/HBaseReadWrite.java    |  2942 --
 .../hive/metastore/hbase/HBaseSchemaTool.java   |   192 -
 .../hadoop/hive/metastore/hbase/HBaseStore.java |  2969 --
 .../hadoop/hive/metastore/hbase/HBaseUtils.java |  1781 -
 .../hive/metastore/hbase/ObjectCache.java       |    81 -
 .../hive/metastore/hbase/PartitionCache.java    |   168 -
 .../metastore/hbase/PartitionKeyComparator.java |   292 -
 .../hbase/SharedStorageDescriptor.java          |   251 -
 .../hadoop/hive/metastore/hbase/StatsCache.java |   325 -
 .../metastore/hbase/TephraHBaseConnection.java  |   127 -
 .../metastore/hbase/VanillaHBaseConnection.java |   137 -
 .../stats/BinaryColumnStatsAggregator.java      |    64 -
 .../stats/BooleanColumnStatsAggregator.java     |    65 -
 .../hbase/stats/ColumnStatsAggregator.java      |    33 -
 .../stats/ColumnStatsAggregatorFactory.java     |   113 -
 .../stats/DecimalColumnStatsAggregator.java     |   371 -
 .../stats/DoubleColumnStatsAggregator.java      |   345 -
 .../hbase/stats/IExtrapolatePartStatus.java     |    47 -
 .../hbase/stats/LongColumnStatsAggregator.java  |   344 -
 .../stats/merge/BinaryColumnStatsMerger.java    |    35 -
 .../stats/merge/BooleanColumnStatsMerger.java   |    35 -
 .../hbase/stats/merge/ColumnStatsMerger.java    |    31 -
 .../stats/merge/ColumnStatsMergerFactory.java   |   123 -
 .../stats/merge/DateColumnStatsMerger.java      |    61 -
 .../stats/merge/DecimalColumnStatsMerger.java   |    63 -
 .../stats/merge/DoubleColumnStatsMerger.java    |    56 -
 .../stats/merge/LongColumnStatsMerger.java      |    56 -
 .../stats/merge/StringColumnStatsMerger.java    |    56 -
 .../metastore/hbase/hbase_metastore_proto.proto |   368 -
 .../hadoop/hive/metastore/hbase/MockUtils.java  |   219 -
 .../hbase/TestHBaseAggregateStatsCache.java     |   316 -
 ...stHBaseAggregateStatsCacheWithBitVector.java |   193 -
 .../TestHBaseAggregateStatsExtrapolation.java   |   721 -
 .../TestHBaseAggregateStatsNDVUniformDist.java  |   589 -
 .../hbase/TestHBaseFilterPlanUtil.java          |   483 -
 .../hive/metastore/hbase/TestHBaseStore.java    |  1978 -
 .../hbase/TestHBaseStoreBitVector.java          |   622 -
 .../metastore/hbase/TestHBaseStoreCached.java   |   365 -
 .../hbase/TestSharedStorageDescriptor.java      |   152 -
 89 files changed, 2018 insertions(+), 70161 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/bin/ext/hbaseimport.sh
----------------------------------------------------------------------
diff --git a/bin/ext/hbaseimport.sh b/bin/ext/hbaseimport.sh
deleted file mode 100644
index 638cdcf..0000000
--- a/bin/ext/hbaseimport.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-THISSERVICE=hbaseimport
-export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
-
-hbaseimport () {
-  CLASS=org.apache.hadoop.hive.metastore.hbase.HBaseImport
-  HIVE_OPTS=''
-  execHiveCmd $CLASS "$@"
-}
-
-hbaseimport_help () {
-  echo "usage ./hive hbaseimport"
-} 

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/bin/ext/hbaseschematool.sh
----------------------------------------------------------------------
diff --git a/bin/ext/hbaseschematool.sh b/bin/ext/hbaseschematool.sh
deleted file mode 100644
index 4d4570a..0000000
--- a/bin/ext/hbaseschematool.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-THISSERVICE=hbaseschematool
-export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
-
-hbaseschematool () {
-  CLASS=org.apache.hadoop.hive.metastore.hbase.HBaseSchemaTool
-  HIVE_OPTS=''
-  execHiveCmd $CLASS "$@"
-}
-
-hbaseschematool_help () {
-  echo "usage ./hive hbaseschematool [-d <dbname>] <cmd>"
-} 

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 14235b5..62a65c9 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -291,21 +291,6 @@ public class HiveConf extends Configuration {
       HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_FULL,
       HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL,
       HiveConf.ConfVars.METASTORE_FASTPATH,
-      HiveConf.ConfVars.METASTORE_HBASE_CATALOG_CACHE_SIZE,
-      HiveConf.ConfVars.METASTORE_HBASE_AGGREGATE_STATS_CACHE_SIZE,
-      HiveConf.ConfVars.METASTORE_HBASE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS,
-      HiveConf.ConfVars.METASTORE_HBASE_AGGREGATE_STATS_CACHE_FALSE_POSITIVE_PROBABILITY,
-      HiveConf.ConfVars.METASTORE_HBASE_AGGREGATE_STATS_CACHE_MAX_VARIANCE,
-      HiveConf.ConfVars.METASTORE_HBASE_CACHE_TIME_TO_LIVE,
-      HiveConf.ConfVars.METASTORE_HBASE_CACHE_MAX_WRITER_WAIT,
-      HiveConf.ConfVars.METASTORE_HBASE_CACHE_MAX_READER_WAIT,
-      HiveConf.ConfVars.METASTORE_HBASE_CACHE_MAX_FULL,
-      HiveConf.ConfVars.METASTORE_HBASE_CACHE_CLEAN_UNTIL,
-      HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS,
-      HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_CACHE_ENTRIES,
-      HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_MEMORY_TTL,
-      HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_INVALIDATOR_FREQUENCY,
-      HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_HBASE_TTL,
       HiveConf.ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS
       };
 
@@ -622,43 +607,6 @@ public class HiveConf extends Configuration {
             "undefined and most likely undesired behavior will result"),
     METASTORE_FS_HANDLER_THREADS_COUNT("hive.metastore.fshandler.threads", 15,
         "Number of threads to be allocated for metastore handler for fs operations."),
-    METASTORE_HBASE_CATALOG_CACHE_SIZE("hive.metastore.hbase.catalog.cache.size", 50000, "Maximum number of " +
-        "objects we will place in the hbase metastore catalog cache.  The objects will be divided up by " +
-        "types that we need to cache."),
-    METASTORE_HBASE_AGGREGATE_STATS_CACHE_SIZE("hive.metastore.hbase.aggregate.stats.cache.size", 10000,
-        "Maximum number of aggregate stats nodes that we will place in the hbase metastore aggregate stats cache."),
-    METASTORE_HBASE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS("hive.metastore.hbase.aggregate.stats.max.partitions", 10000,
-        "Maximum number of partitions that are aggregated per cache node."),
-    METASTORE_HBASE_AGGREGATE_STATS_CACHE_FALSE_POSITIVE_PROBABILITY("hive.metastore.hbase.aggregate.stats.false.positive.probability",
-        (float) 0.01, "Maximum false positive probability for the Bloom Filter used in each aggregate stats cache node (default 1%)."),
-    METASTORE_HBASE_AGGREGATE_STATS_CACHE_MAX_VARIANCE("hive.metastore.hbase.aggregate.stats.max.variance", (float) 0.1,
-        "Maximum tolerable variance in number of partitions between a cached node and our request (default 10%)."),
-    METASTORE_HBASE_CACHE_TIME_TO_LIVE("hive.metastore.hbase.cache.ttl", "600s", new TimeValidator(TimeUnit.SECONDS),
-        "Number of seconds for a cached node to be active in the cache before they become stale."),
-    METASTORE_HBASE_CACHE_MAX_WRITER_WAIT("hive.metastore.hbase.cache.max.writer.wait", "5000ms", new TimeValidator(TimeUnit.MILLISECONDS),
-        "Number of milliseconds a writer will wait to acquire the writelock before giving up."),
-    METASTORE_HBASE_CACHE_MAX_READER_WAIT("hive.metastore.hbase.cache.max.reader.wait", "1000ms", new TimeValidator(TimeUnit.MILLISECONDS),
-         "Number of milliseconds a reader will wait to acquire the readlock before giving up."),
-    METASTORE_HBASE_CACHE_MAX_FULL("hive.metastore.hbase.cache.max.full", (float) 0.9,
-         "Maximum cache full % after which the cache cleaner thread kicks in."),
-    METASTORE_HBASE_CACHE_CLEAN_UNTIL("hive.metastore.hbase.cache.clean.until", (float) 0.8,
-          "The cleaner thread cleans until cache reaches this % full size."),
-    METASTORE_HBASE_CONNECTION_CLASS("hive.metastore.hbase.connection.class",
-        "org.apache.hadoop.hive.metastore.hbase.VanillaHBaseConnection",
-        "Class used to connection to HBase"),
-    METASTORE_HBASE_AGGR_STATS_CACHE_ENTRIES("hive.metastore.hbase.aggr.stats.cache.entries",
-        10000, "How many in stats objects to cache in memory"),
-    METASTORE_HBASE_AGGR_STATS_MEMORY_TTL("hive.metastore.hbase.aggr.stats.memory.ttl", "60s",
-        new TimeValidator(TimeUnit.SECONDS),
-        "Number of seconds stats objects live in memory after they are read from HBase."),
-    METASTORE_HBASE_AGGR_STATS_INVALIDATOR_FREQUENCY(
-        "hive.metastore.hbase.aggr.stats.invalidator.frequency", "5s",
-        new TimeValidator(TimeUnit.SECONDS),
-        "How often the stats cache scans its HBase entries and looks for expired entries"),
-    METASTORE_HBASE_AGGR_STATS_HBASE_TTL("hive.metastore.hbase.aggr.stats.hbase.ttl", "604800s",
-        new TimeValidator(TimeUnit.SECONDS),
-        "Number of seconds stats entries live in HBase cache after they are created.  They may be" +
-            " invalided by updates or partition drops before this.  Default is one week."),
     METASTORE_HBASE_FILE_METADATA_THREADS("hive.metastore.hbase.file.metadata.threads", 1,
         "Number of threads to use to read file metadata in background to cache it."),
 

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
deleted file mode 100644
index d4cd818..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Integration tests with HBase Mini-cluster for HBaseStore
- */
-public class HBaseIntegrationTests {
-
-  private static final Logger LOG = LoggerFactory.getLogger(HBaseIntegrationTests.class.getName());
-
-  protected static HBaseTestingUtility utility;
-  protected static HBaseAdmin admin;
-  protected static Map<String, String> emptyParameters = new HashMap<>();
-  protected static HiveConf conf;
-
-  protected HBaseStore store;
-  protected Driver driver;
-
-  protected static void startMiniCluster() throws Exception {
-    String connectionClassName =
-        System.getProperty(HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS.varname);
-    boolean testingTephra =
-        connectionClassName != null && connectionClassName.equals(TephraHBaseConnection.class.getName());
-    if (testingTephra) {
-      LOG.info("Testing with Tephra");
-    }
-    Configuration hbaseConf = HBaseConfiguration.create();
-    hbaseConf.setInt("hbase.master.info.port", -1);
-    utility = new HBaseTestingUtility(hbaseConf);
-    utility.startMiniCluster();
-    conf = new HiveConf(utility.getConfiguration(), HBaseIntegrationTests.class);
-    admin = utility.getHBaseAdmin();
-    HBaseStoreTestUtil.initHBaseMetastore(admin, null);
-  }
-
-  protected static void shutdownMiniCluster() throws Exception {
-    utility.shutdownMiniCluster();
-  }
-
-  protected void setupConnection() throws IOException {
-
-  }
-
-  protected void setupDriver() {
-    // This chicanery is necessary to make the driver work.  Hive tests need the pfile file
-    // system, while the hbase one uses something else.  So first make sure we've configured our
-    // hbase connection, then get a new config file and populate it as desired.
-    HBaseReadWrite.setConf(conf);
-    conf = new HiveConf();
-    conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
-    conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
-        "org.apache.hadoop.hive.metastore.hbase.HBaseStore");
-    conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true);
-    conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
-    // Setup so we can test SQL standard auth
-    conf.setBoolVar(HiveConf.ConfVars.HIVE_TEST_AUTHORIZATION_SQLSTD_HS2_MODE, true);
-    conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
-        SQLStdHiveAuthorizerFactoryForTest.class.getName());
-    conf.setVar(HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
-        SessionStateConfigUserAuthenticator.class.getName());
-    conf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED, true);
-    conf.setVar(HiveConf.ConfVars.USERS_IN_ADMIN_ROLE, System.getProperty("user.name"));
-    conf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE,"nonstrict");
-    //HBaseReadWrite.setTestConnection(hconn);
-
-    SessionState.start(new CliSessionState(conf));
-    driver = new Driver(conf);
-  }
-
-  protected void setupHBaseStore() {
-    // Turn off caching, as we want to test actual interaction with HBase
-    conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
-    store = new HBaseStore();
-    store.setConf(conf);
-  }
-
-}
-

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggrStatsCacheIntegration.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggrStatsCacheIntegration.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggrStatsCacheIntegration.java
deleted file mode 100644
index 51d96dd..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggrStatsCacheIntegration.java
+++ /dev/null
@@ -1,691 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Integration tests with HBase Mini-cluster for HBaseStore
- */
-public class TestHBaseAggrStatsCacheIntegration extends HBaseIntegrationTests {
-
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreIntegration.class.getName());
-
-  @Rule public ExpectedException thrown = ExpectedException.none();
-
-  @BeforeClass
-  public static void startup() throws Exception {
-    HBaseIntegrationTests.startMiniCluster();
-  }
-
-  @AfterClass
-  public static void shutdown() throws Exception {
-    HBaseIntegrationTests.shutdownMiniCluster();
-  }
-
-  @Before
-  public void setup() throws IOException {
-    setupConnection();
-    setupHBaseStore();
-    store.backdoor().getStatsCache().resetCounters();
-  }
-
-  private static interface Checker {
-    void checkStats(AggrStats aggrStats) throws Exception;
-  }
-
-  @Test
-  public void hit() throws Exception {
-    String dbName = "default";
-    String tableName = "hit";
-    List<String> partVals1 = Arrays.asList("today");
-    List<String> partVals2 = Arrays.asList("yesterday");
-    long now = System.currentTimeMillis();
-
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1", "boolean", "nocomment"));
-    cols.add(new FieldSchema("col2", "varchar", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String>emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
-        Collections.<String, String>emptyMap(), null, null, null);
-    store.createTable(table);
-
-    for (List<String> partVals : Arrays.asList(partVals1, partVals2)) {
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/hit/ds=" + partVals.get(0));
-      Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String>emptyMap());
-      store.addPartition(part);
-
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVals.get(0));
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col1");
-      obj.setColType("boolean");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
-      bcsd.setNumFalses(10);
-      bcsd.setNumTrues(20);
-      bcsd.setNumNulls(30);
-      data.setBooleanStats(bcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-
-      obj = new ColumnStatisticsObj();
-      obj.setColName("col2");
-      obj.setColType("varchar");
-      data = new ColumnStatisticsData();
-      StringColumnStatsData scsd = new StringColumnStatsData();
-      scsd.setAvgColLen(10.3);
-      scsd.setMaxColLen(2000);
-      scsd.setNumNulls(3);
-      scsd.setNumDVs(12342);
-      data.setStringStats(scsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-
-      store.updatePartitionColumnStatistics(cs, partVals);
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(2, aggrStats.getPartsFound());
-        Assert.assertEquals(2, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1", cso.getColName());
-        Assert.assertEquals("boolean", cso.getColType());
-        BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
-        Assert.assertEquals(20, bcsd.getNumFalses());
-        Assert.assertEquals(40, bcsd.getNumTrues());
-        Assert.assertEquals(60, bcsd.getNumNulls());
-
-        cso = aggrStats.getColStats().get(1);
-        Assert.assertEquals("col2", cso.getColName());
-        Assert.assertEquals("varchar", cso.getColType());
-        StringColumnStatsData scsd = cso.getStatsData().getStringStats();
-        Assert.assertEquals(10.3, scsd.getAvgColLen(), 0.1);
-        Assert.assertEquals(2000, scsd.getMaxColLen());
-        Assert.assertEquals(6, scsd.getNumNulls());
-        Assert.assertEquals(12342, scsd.getNumDVs());
-      }
-    };
-
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1", "col2"));
-    statChecker.checkStats(aggrStats);
-
-    // Check that we had to build it from the stats
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
-    // Call again, this time it should come from memory.  Also, reverse the name order this time
-    // to assure that we still hit.
-    aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1", "col2"));
-    statChecker.checkStats(aggrStats);
-
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(4, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
-    store.backdoor().getStatsCache().flushMemory();
-    // Call again, this time it should come from hbase
-    aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1", "col2"));
-    statChecker.checkStats(aggrStats);
-
-    Assert.assertEquals(2, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(6, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-  }
-
-  @Test
-  public void someWithStats() throws Exception {
-    String dbName = "default";
-    String tableName = "psws";
-    List<String> partVals1 = Arrays.asList("today");
-    List<String> partVals2 = Arrays.asList("yesterday");
-    long now = System.currentTimeMillis();
-
-    List<FieldSchema> cols = new ArrayList<>();
-    cols.add(new FieldSchema("col1", "long", "nocomment"));
-    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-        serde, null, null, Collections.<String, String>emptyMap());
-    List<FieldSchema> partCols = new ArrayList<>();
-    partCols.add(new FieldSchema("ds", "string", ""));
-    Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
-        Collections.<String, String>emptyMap(), null, null, null);
-    store.createTable(table);
-
-    boolean first = true;
-    for (List<String> partVals : Arrays.asList(partVals1, partVals2)) {
-      StorageDescriptor psd = new StorageDescriptor(sd);
-      psd.setLocation("file:/tmp/default/psws/ds=" + partVals.get(0));
-      Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
-          Collections.<String, String>emptyMap());
-      store.addPartition(part);
-
-      if (first) {
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVals.get(0));
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col1");
-        obj.setColType("long");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        LongColumnStatsData lcsd = new LongColumnStatsData();
-        lcsd.setHighValue(192L);
-        lcsd.setLowValue(-20L);
-        lcsd.setNumNulls(30);
-        lcsd.setNumDVs(32);
-        data.setLongStats(lcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-
-        store.updatePartitionColumnStatistics(cs, partVals);
-        first = false;
-      }
-    }
-
-    Checker statChecker = new Checker() {
-      @Override
-      public void checkStats(AggrStats aggrStats) throws Exception {
-        Assert.assertEquals(1, aggrStats.getPartsFound());
-        Assert.assertEquals(1, aggrStats.getColStatsSize());
-        ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-        Assert.assertEquals("col1", cso.getColName());
-        Assert.assertEquals("long", cso.getColType());
-        LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
-        Assert.assertEquals(192L, lcsd.getHighValue());
-        Assert.assertEquals(-20L, lcsd.getLowValue());
-        Assert.assertEquals(30, lcsd.getNumNulls());
-        Assert.assertEquals(32, lcsd.getNumDVs());
-      }
-    };
-
-    AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    // Check that we had to build it from the stats
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
-    // Call again, this time it should come from memory.  Also, reverse the name order this time
-    // to assure that we still hit.
-    aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
-    store.backdoor().getStatsCache().flushMemory();
-    // Call again, this time it should come from hbase
-    aggrStats = store.get_aggr_stats_for(dbName, tableName,
-        Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-    statChecker.checkStats(aggrStats);
-
-    Assert.assertEquals(1, store.backdoor().getStatsCache().hbaseHits.getCnt());
-    Assert.assertEquals(3, store.backdoor().getStatsCache().totalGets.getCnt());
-    Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-  }
-
-  @Test
-  public void invalidation() throws Exception {
-    try {
-      String dbName = "default";
-      String tableName = "invalidation";
-      List<String> partVals1 = Arrays.asList("today");
-      List<String> partVals2 = Arrays.asList("yesterday");
-      List<String> partVals3 = Arrays.asList("tomorrow");
-      long now = System.currentTimeMillis();
-
-      List<FieldSchema> cols = new ArrayList<>();
-      cols.add(new FieldSchema("col1", "boolean", "nocomment"));
-      SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-      StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-          serde, null, null, Collections.<String, String>emptyMap());
-      List<FieldSchema> partCols = new ArrayList<>();
-      partCols.add(new FieldSchema("ds", "string", ""));
-      Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-          Collections.<String, String>emptyMap(), null, null, null);
-      store.createTable(table);
-
-      for (List<String> partVals : Arrays.asList(partVals1, partVals2, partVals3)) {
-        StorageDescriptor psd = new StorageDescriptor(sd);
-        psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
-        Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
-            Collections.<String, String>emptyMap());
-        store.addPartition(part);
-
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVals.get(0));
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col1");
-        obj.setColType("boolean");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
-        bcsd.setNumFalses(10);
-        bcsd.setNumTrues(20);
-        bcsd.setNumNulls(30);
-        data.setBooleanStats(bcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-
-        store.updatePartitionColumnStatistics(cs, partVals);
-      }
-
-      Checker statChecker = new Checker() {
-        @Override
-        public void checkStats(AggrStats aggrStats) throws Exception {
-          Assert.assertEquals(2, aggrStats.getPartsFound());
-          Assert.assertEquals(1, aggrStats.getColStatsSize());
-          ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-          Assert.assertEquals("col1", cso.getColName());
-          Assert.assertEquals("boolean", cso.getColType());
-          BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
-          Assert.assertEquals(20, bcsd.getNumFalses());
-          Assert.assertEquals(40, bcsd.getNumTrues());
-          Assert.assertEquals(60, bcsd.getNumNulls());
-        }
-      };
-
-      AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-      statChecker.checkStats(aggrStats);
-
-      // Check that we had to build it from the stats
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
-      // Call again, this time it should come from memory.  Also, reverse the name order this time
-      // to assure that we still hit.
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-      statChecker.checkStats(aggrStats);
-
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
-      // Now call a different combination to get it in memory too
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
-      statChecker.checkStats(aggrStats);
-
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(3, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
-      statChecker.checkStats(aggrStats);
-
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(4, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
-      // wake the invalidator and check again to make sure it isn't too aggressive about
-      // removing our stuff.
-      store.backdoor().getStatsCache().wakeInvalidator();
-
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
-      statChecker.checkStats(aggrStats);
-
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(5, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
-      // Update statistics for 'tomorrow'
-      ColumnStatistics cs = new ColumnStatistics();
-      ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-      desc.setLastAnalyzed(now);
-      desc.setPartName("ds=" + partVals3.get(0));
-      cs.setStatsDesc(desc);
-      ColumnStatisticsObj obj = new ColumnStatisticsObj();
-      obj.setColName("col1");
-      obj.setColType("boolean");
-      ColumnStatisticsData data = new ColumnStatisticsData();
-      BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
-      bcsd.setNumFalses(100);
-      bcsd.setNumTrues(200);
-      bcsd.setNumNulls(300);
-      data.setBooleanStats(bcsd);
-      obj.setStatsData(data);
-      cs.addToStatsObj(obj);
-
-      Checker afterUpdate = new Checker() {
-        @Override
-        public void checkStats(AggrStats aggrStats) throws Exception {
-          Assert.assertEquals(2, aggrStats.getPartsFound());
-          Assert.assertEquals(1, aggrStats.getColStatsSize());
-          ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-          Assert.assertEquals("col1", cso.getColName());
-          Assert.assertEquals("boolean", cso.getColType());
-          BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
-          Assert.assertEquals(110, bcsd.getNumFalses());
-          Assert.assertEquals(220, bcsd.getNumTrues());
-          Assert.assertEquals(330, bcsd.getNumNulls());
-        }
-      };
-
-      store.updatePartitionColumnStatistics(cs, partVals3);
-
-      store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
-      store.backdoor().getStatsCache().wakeInvalidator();
-
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
-      afterUpdate.checkStats(aggrStats);
-
-      // Check that we missed, which means this aggregate was dropped from the cache.
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(6, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
-
-      // Check that our other aggregate is still in the cache.
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-      statChecker.checkStats(aggrStats);
-
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(7, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
-
-      // Drop 'yesterday', so our first aggregate should be dumped from memory and hbase
-      store.dropPartition(dbName, tableName, partVals2);
-
-      store.backdoor().getStatsCache().wakeInvalidator();
-
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-      new Checker() {
-        @Override
-        public void checkStats(AggrStats aggrStats) throws Exception {
-          Assert.assertEquals(1, aggrStats.getPartsFound());
-          Assert.assertEquals(1, aggrStats.getColStatsSize());
-          ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
-          Assert.assertEquals("col1", cso.getColName());
-          Assert.assertEquals("boolean", cso.getColType());
-          BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
-          Assert.assertEquals(10, bcsd.getNumFalses());
-          Assert.assertEquals(20, bcsd.getNumTrues());
-          Assert.assertEquals(30, bcsd.getNumNulls());
-        }
-      }.checkStats(aggrStats);
-
-      // Check that we missed, which means this aggregate was dropped from the cache.
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(8, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(4, store.backdoor().getStatsCache().misses.getCnt());
-
-      // Check that our other aggregate is still in the cache.
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
-      afterUpdate.checkStats(aggrStats);
-
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(9, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(4, store.backdoor().getStatsCache().misses.getCnt());
-    } finally {
-      store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
-      store.backdoor().getStatsCache().setMaxTimeInCache(500000);
-      store.backdoor().getStatsCache().wakeInvalidator();
-    }
-  }
-
-  @Test
-  public void alterInvalidation() throws Exception {
-    try {
-      String dbName = "default";
-      String tableName = "ai";
-      List<String> partVals1 = Arrays.asList("today");
-      List<String> partVals2 = Arrays.asList("yesterday");
-      List<String> partVals3 = Arrays.asList("tomorrow");
-      long now = System.currentTimeMillis();
-
-      List<FieldSchema> cols = new ArrayList<>();
-      cols.add(new FieldSchema("col1", "boolean", "nocomment"));
-      SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-      StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-          serde, null, null, Collections.<String, String>emptyMap());
-      List<FieldSchema> partCols = new ArrayList<>();
-      partCols.add(new FieldSchema("ds", "string", ""));
-      Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-          Collections.<String, String>emptyMap(), null, null, null);
-      store.createTable(table);
-
-      Partition[] partitions = new Partition[3];
-      int partnum = 0;
-      for (List<String> partVals : Arrays.asList(partVals1, partVals2, partVals3)) {
-        StorageDescriptor psd = new StorageDescriptor(sd);
-        psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
-        Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
-            Collections.<String, String>emptyMap());
-        partitions[partnum++] = part;
-        store.addPartition(part);
-
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVals.get(0));
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col1");
-        obj.setColType("boolean");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
-        bcsd.setNumFalses(10);
-        bcsd.setNumTrues(20);
-        bcsd.setNumNulls(30);
-        data.setBooleanStats(bcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-
-        store.updatePartitionColumnStatistics(cs, partVals);
-      }
-
-      AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=today", "ds=tomorrow"), Arrays.asList("col1"));
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-
-      // Check that we had to build it from the stats
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
-      // wake the invalidator and check again to make sure it isn't too aggressive about
-      // removing our stuff.
-      store.backdoor().getStatsCache().wakeInvalidator();
-
-      Partition newPart = new Partition(partitions[2]);
-      newPart.setLastAccessTime((int)System.currentTimeMillis());
-      store.alterPartition(dbName, tableName, partVals3, newPart);
-
-      store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
-      store.backdoor().getStatsCache().wakeInvalidator();
-
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
-
-      // Check that we missed, which means this aggregate was dropped from the cache.
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(3, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
-
-      // Check that our other aggregate is still in the cache.
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(4, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
-    } finally {
-      store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
-      store.backdoor().getStatsCache().setMaxTimeInCache(500000);
-      store.backdoor().getStatsCache().wakeInvalidator();
-    }
-  }
-
-  @Test
-  public void altersInvalidation() throws Exception {
-    try {
-      String dbName = "default";
-      String tableName = "asi";
-      List<String> partVals1 = Arrays.asList("today");
-      List<String> partVals2 = Arrays.asList("yesterday");
-      List<String> partVals3 = Arrays.asList("tomorrow");
-      long now = System.currentTimeMillis();
-
-      List<FieldSchema> cols = new ArrayList<>();
-      cols.add(new FieldSchema("col1", "boolean", "nocomment"));
-      SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-      StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-          serde, null, null, Collections.<String, String>emptyMap());
-      List<FieldSchema> partCols = new ArrayList<>();
-      partCols.add(new FieldSchema("ds", "string", ""));
-      Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
-          Collections.<String, String>emptyMap(), null, null, null);
-      store.createTable(table);
-
-      Partition[] partitions = new Partition[3];
-      int partnum = 0;
-      for (List<String> partVals : Arrays.asList(partVals1, partVals2, partVals3)) {
-        StorageDescriptor psd = new StorageDescriptor(sd);
-        psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
-        Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
-            Collections.<String, String>emptyMap());
-        partitions[partnum++] = part;
-        store.addPartition(part);
-
-        ColumnStatistics cs = new ColumnStatistics();
-        ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
-        desc.setLastAnalyzed(now);
-        desc.setPartName("ds=" + partVals.get(0));
-        cs.setStatsDesc(desc);
-        ColumnStatisticsObj obj = new ColumnStatisticsObj();
-        obj.setColName("col1");
-        obj.setColType("boolean");
-        ColumnStatisticsData data = new ColumnStatisticsData();
-        BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
-        bcsd.setNumFalses(10);
-        bcsd.setNumTrues(20);
-        bcsd.setNumNulls(30);
-        data.setBooleanStats(bcsd);
-        obj.setStatsData(data);
-        cs.addToStatsObj(obj);
-
-        store.updatePartitionColumnStatistics(cs, partVals);
-      }
-
-      AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=today", "ds=tomorrow"), Arrays.asList("col1"));
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-
-      // Check that we had to build it from the stats
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
-      // wake the invalidator and check again to make sure it isn't too aggressive about
-      // removing our stuff.
-      store.backdoor().getStatsCache().wakeInvalidator();
-
-      Partition[] newParts = new Partition[2];
-      newParts[0] = new Partition(partitions[0]);
-      newParts[0].setLastAccessTime((int)System.currentTimeMillis());
-      newParts[1] = new Partition(partitions[2]);
-      newParts[1].setLastAccessTime((int) System.currentTimeMillis());
-      store.alterPartitions(dbName, tableName, Arrays.asList(partVals1, partVals3),
-          Arrays.asList(newParts));
-
-      store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
-      store.backdoor().getStatsCache().wakeInvalidator();
-
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
-
-      // Check that we missed, which means this aggregate was dropped from the cache.
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(3, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
-
-      // Check that our other aggregate got dropped too
-      aggrStats = store.get_aggr_stats_for(dbName, tableName,
-          Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-
-      Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
-      Assert.assertEquals(4, store.backdoor().getStatsCache().totalGets.getCnt());
-      Assert.assertEquals(4, store.backdoor().getStatsCache().misses.getCnt());
-    } finally {
-      store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
-      store.backdoor().getStatsCache().setMaxTimeInCache(500000);
-      store.backdoor().getStatsCache().wakeInvalidator();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java
deleted file mode 100644
index b1d3174..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java
+++ /dev/null
@@ -1,775 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.TestObjectStore;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.ResourceType;
-import org.apache.hadoop.hive.metastore.api.ResourceUri;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Test that import from an RDBMS based metastore works
- */
-public class TestHBaseImport extends HBaseIntegrationTests {
-
-  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseImport.class.getName());
-
-  private static final String[] tableNames = new String[] {"allnonparttable", "allparttable"};
-  private static final String[] partVals = new String[] {"na", "emea", "latam", "apac"};
-  private static final String[] funcNames = new String[] {"allfunc1", "allfunc2"};
-  private static final String[] indexNames = new String[] {"allindex1", "allindex2"};
-  private static final String[] pkNames = new String[] {"allnonparttable_pk", "allparttable_pk"};
-  private static final String[] fkNames = new String[] {"", "allparttable_fk"};
-
-  private static final List<Integer> masterKeySeqs = new ArrayList<Integer>();
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
-
-  @BeforeClass
-  public static void startup() throws Exception {
-    HBaseIntegrationTests.startMiniCluster();
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-    TestObjectStore.dropAllStoreObjects(rdbms);
-  }
-
-  @AfterClass
-  public static void shutdown() throws Exception {
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-    TestObjectStore.dropAllStoreObjects(rdbms);
-    for (int seq : masterKeySeqs) {
-      rdbms.removeMasterKey(seq);
-    }
-    HBaseIntegrationTests.shutdownMiniCluster();
-  }
-
-  @Before
-  public void setup() throws IOException {
-    setupConnection();
-    setupHBaseStore();
-  }
-
-  @Test
-  public void importAll() throws Exception {
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"alldb1", "alldb2"};
-    String[] roles = new String[] {"allrole1", "allrole2"};
-    String[] tokenIds = new String[] {"alltokenid1", "alltokenid2"};
-    String[] tokens = new String[] {"alltoken1", "alltoken2"};
-    String[] masterKeys = new String[] {"allmk1", "allmk2"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
-    int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
-    int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
-
-    HBaseImport importer = new HBaseImport("-a");
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    for (int i = 0; i < roles.length; i++) {
-      Role role = store.getRole(roles[i]);
-      Assert.assertNotNull(role);
-      Assert.assertEquals(roles[i], role.getRoleName());
-    }
-    // Make sure there aren't any extra roles
-    Assert.assertEquals(baseNumRoles + 2, store.listRoleNames().size());
-
-    for (int i = 0; i < dbNames.length; i++) {
-      Database db = store.getDatabase(dbNames[i]);
-      Assert.assertNotNull(db);
-      // check one random value in the db rather than every value
-      Assert.assertEquals("file:/tmp", db.getLocationUri());
-
-      Table table = store.getTable(db.getName(), tableNames[0]);
-      Assert.assertNotNull(table);
-      Assert.assertEquals(now, table.getLastAccessTime());
-      Assert.assertEquals("input", table.getSd().getInputFormat());
-
-      table = store.getTable(db.getName(), tableNames[1]);
-      Assert.assertNotNull(table);
-
-      for (int j = 0; j < partVals.length; j++) {
-        Partition part = store.getPartition(dbNames[i], tableNames[1], Arrays.asList(partVals[j]));
-        Assert.assertNotNull(part);
-        Assert.assertEquals("file:/tmp/region=" + partVals[j], part.getSd().getLocation());
-      }
-
-      Assert.assertEquals(4, store.getPartitions(dbNames[i], tableNames[1], -1).size());
-      // Including two index table
-      Assert.assertEquals(4, store.getAllTables(dbNames[i]).size());
-
-      Assert.assertEquals(2, store.getIndexes(dbNames[i], tableNames[0], -1).size());
-      Assert.assertEquals(0, store.getIndexes(dbNames[i], tableNames[1], -1).size());
-
-      Assert.assertEquals(2, store.getFunctions(dbNames[i], "*").size());
-      for (int j = 0; j < funcNames.length; j++) {
-        Assert.assertNotNull(store.getFunction(dbNames[i], funcNames[j]));
-      }
-    }
-
-    Assert.assertEquals(baseNumDbs + 2, store.getAllDatabases().size());
-
-    // I can't test total number of tokens or master keys because the import grabs all and copies
-    // them, which means it grabs the ones imported by importSecurity test (if it's already run).
-    // Depending on it already running would make the tests order dependent, which junit doesn't
-    // guarantee.
-    for (int i = 0; i < tokenIds.length; i++) {
-      Assert.assertEquals(tokens[i], store.getToken(tokenIds[i]));
-    }
-    String[] hbaseKeys = store.getMasterKeys();
-    Set<String> keys = new HashSet<>(Arrays.asList(hbaseKeys));
-    for (int i = 0; i < masterKeys.length; i++) {
-      Assert.assertTrue(keys.contains(masterKeys[i]));
-    }
-  }
-
-  @Test
-  public void importOneDb() throws Exception {
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"onedbdb1", "onedbdb2"};
-    String[] roles = new String[] {"onedbrole1", "onedbrole2"};
-    String[] tokenIds = new String[] {"onedbtokenid1", "onedbtokenid2"};
-    String[] tokens = new String[] {"onedbtoken1", "onedbtoken2"};
-    String[] masterKeys = new String[] {"onedbmk1", "onedbmk2"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
-    int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
-    int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
-    int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
-        store.getAllTokenIdentifiers().size();
-    int baseNumKeys =  store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
-    HBaseImport importer = new HBaseImport("-d", dbNames[0]);
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    // Make sure there aren't any extra roles
-    Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
-    Database db = store.getDatabase(dbNames[0]);
-    Assert.assertNotNull(db);
-    // check one random value in the db rather than every value
-    Assert.assertEquals("file:/tmp", db.getLocationUri());
-
-    Table table = store.getTable(db.getName(), tableNames[0]);
-    Assert.assertNotNull(table);
-    Assert.assertEquals(now, table.getLastAccessTime());
-    Assert.assertEquals("input", table.getSd().getInputFormat());
-
-    table = store.getTable(db.getName(), tableNames[1]);
-    Assert.assertNotNull(table);
-
-    for (int j = 0; j < partVals.length; j++) {
-      Partition part = store.getPartition(dbNames[0], tableNames[1], Arrays.asList(partVals[j]));
-      Assert.assertNotNull(part);
-      Assert.assertEquals("file:/tmp/region=" + partVals[j], part.getSd().getLocation());
-    }
-
-    Assert.assertEquals(4, store.getPartitions(dbNames[0], tableNames[1], -1).size());
-    // Including two index table
-    Assert.assertEquals(4, store.getAllTables(dbNames[0]).size());
-
-    Assert.assertEquals(2, store.getIndexes(dbNames[0], tableNames[0], -1).size());
-    Assert.assertEquals(0, store.getIndexes(dbNames[0], tableNames[1], -1).size());
-
-    Assert.assertEquals(2, store.getFunctions(dbNames[0], "*").size());
-    for (int j = 0; j < funcNames.length; j++) {
-      Assert.assertNotNull(store.getFunction(dbNames[0], funcNames[j]));
-    }
-
-    Assert.assertEquals(baseNumDbs + 1, store.getAllDatabases().size());
-
-    Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
-    String[] hbaseKeys = store.getMasterKeys();
-    Assert.assertEquals(baseNumKeys, hbaseKeys.length);
-
-    // Have to do this last as it will throw an exception
-    thrown.expect(NoSuchObjectException.class);
-    store.getDatabase(dbNames[1]);
-  }
-
-  @Test
-  public void importOneFunc() throws Exception {
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"onefuncdb1", "onefuncdb2"};
-    String[] roles = new String[] {"onefuncrole1", "onefuncrole2"};
-    String[] tokenIds = new String[] {"onefunctokenid1", "onefunctokenid2"};
-    String[] tokens = new String[] {"onefunctoken1", "onefunctoken2"};
-    String[] masterKeys = new String[] {"onefuncmk1", "onefuncmk2"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
-    int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
-    int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
-    int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
-        store.getAllTokenIdentifiers().size();
-    int baseNumKeys =  store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
-    // Create the database so I can put the function in it.
-    store.createDatabase(
-        new Database(dbNames[0], "no description", "file:/tmp", emptyParameters));
-
-    HBaseImport importer = new HBaseImport("-f", dbNames[0] + "." + funcNames[0]);
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    // Make sure there aren't any extra roles
-    Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
-    Database db = store.getDatabase(dbNames[0]);
-    Assert.assertNotNull(db);
-
-    Assert.assertEquals(0, store.getAllTables(dbNames[0]).size());
-    Assert.assertEquals(1, store.getFunctions(dbNames[0], "*").size());
-    Assert.assertNotNull(store.getFunction(dbNames[0], funcNames[0]));
-    Assert.assertNull(store.getFunction(dbNames[0], funcNames[1]));
-
-    Assert.assertEquals(baseNumDbs + 1, store.getAllDatabases().size());
-
-    Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
-    String[] hbaseKeys = store.getMasterKeys();
-    Assert.assertEquals(baseNumKeys, hbaseKeys.length);
-  }
-
-  @Test
-  public void importOneTableNonPartitioned() throws Exception {
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"onetabdb1", "onetabdb2"};
-    String[] roles = new String[] {"onetabrole1", "onetabrole2"};
-    String[] tokenIds = new String[] {"onetabtokenid1", "onetabtokenid2"};
-    String[] tokens = new String[] {"onetabtoken1", "onetabtoken2"};
-    String[] masterKeys = new String[] {"onetabmk1", "onetabmk2"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
-    int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
-    int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
-    int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
-        store.getAllTokenIdentifiers().size();
-    int baseNumKeys =  store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
-    // Create the database so I can put the table in it.
-    store.createDatabase(
-        new Database(dbNames[0], "no description", "file:/tmp", emptyParameters));
-
-    HBaseImport importer = new HBaseImport("-t", dbNames[0] + "." + tableNames[0]);
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    // Make sure there aren't any extra roles
-    Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
-    Database db = store.getDatabase(dbNames[0]);
-    Assert.assertNotNull(db);
-
-    Table table = store.getTable(db.getName(), tableNames[0]);
-    Assert.assertNotNull(table);
-    Assert.assertEquals(1, store.getAllTables(db.getName()).size());
-    Assert.assertNull(store.getTable(db.getName(), tableNames[1]));
-
-    List<Index> indexes = store.getIndexes(db.getName(), tableNames[0], -1);
-    Assert.assertEquals(2, indexes.size());
-
-    Assert.assertEquals(0, store.getFunctions(dbNames[0], "*").size());
-    Assert.assertEquals(baseNumDbs + 1, store.getAllDatabases().size());
-
-    Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
-    String[] hbaseKeys = store.getMasterKeys();
-    Assert.assertEquals(baseNumKeys, hbaseKeys.length);
-  }
-
-  @Test
-  public void importTablesWithConstraints() throws Exception {
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"onetabwcdb1", "onetabwcdb2"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    setupObjectStore(rdbms, dbNames, now, true);
-
-    // Create the database so I can put the table in it.
-    store.createDatabase(
-        new Database(dbNames[0], "no description", "file:/tmp", emptyParameters));
-
-    HBaseImport importer = new HBaseImport("-d", dbNames[0]);
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    Database db = store.getDatabase(dbNames[0]);
-    Assert.assertNotNull(db);
-
-    Table table = store.getTable(db.getName(), tableNames[1]);
-    Assert.assertNotNull(table);
-
-    List<SQLPrimaryKey> pk = store.getPrimaryKeys(dbNames[0], tableNames[1]);
-    Assert.assertNotNull(pk);
-    Assert.assertEquals(1, pk.size());
-    Assert.assertEquals(dbNames[0], pk.get(0).getTable_db());
-    Assert.assertEquals(tableNames[1], pk.get(0).getTable_name());
-    Assert.assertEquals(0, pk.get(0).getKey_seq());
-    Assert.assertEquals("col1", pk.get(0).getColumn_name());
-    Assert.assertEquals(dbNames[0] + "_" + pkNames[1], pk.get(0).getPk_name());
-    Assert.assertTrue(pk.get(0).isEnable_cstr());
-    Assert.assertFalse(pk.get(0).isValidate_cstr());
-    Assert.assertTrue(pk.get(0).isRely_cstr());
-
-    List<SQLForeignKey> fk =
-        store.getForeignKeys(dbNames[0], tableNames[0], dbNames[0], tableNames[1]);
-    Assert.assertNotNull(fk);
-    Assert.assertEquals(1, fk.size());
-    Assert.assertEquals(dbNames[0], fk.get(0).getPktable_db());
-    Assert.assertEquals(tableNames[0], fk.get(0).getPktable_name());
-    Assert.assertEquals("col1", fk.get(0).getPkcolumn_name());
-    Assert.assertEquals(dbNames[0], fk.get(0).getFktable_db());
-    Assert.assertEquals(tableNames[1], fk.get(0).getFktable_name());
-    Assert.assertEquals("col1", fk.get(0).getFkcolumn_name());
-    Assert.assertEquals(0, fk.get(0).getKey_seq());
-    Assert.assertEquals(1, fk.get(0).getUpdate_rule());
-    Assert.assertEquals(2, fk.get(0).getDelete_rule());
-    Assert.assertEquals(dbNames[0] + "_" + fkNames[1], fk.get(0).getFk_name());
-    Assert.assertTrue(pk.get(0).isEnable_cstr());
-    Assert.assertFalse(pk.get(0).isValidate_cstr());
-    Assert.assertTrue(pk.get(0).isRely_cstr());
-
-
-  }
-
-  @Test
-  public void importOneTablePartitioned() throws Exception {
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"onetabpartdb1", "onetabpartodb2"};
-    String[] roles = new String[] {"onetabpartorole1", "onetabpartorole2"};
-    String[] tokenIds = new String[] {"onetabpartotokenid1", "onetabpartotokenid2"};
-    String[] tokens = new String[] {"onetabpartotoken1", "onetabpartotoken2"};
-    String[] masterKeys = new String[] {"onetabpartomk1", "onetabpartomk2"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
-    int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
-    int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
-    int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
-        store.getAllTokenIdentifiers().size();
-    int baseNumKeys =  store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
-    // Create the database so I can put the table in it.
-    store.createDatabase(
-        new Database(dbNames[0], "no description", "file:/tmp", emptyParameters));
-
-    HBaseImport importer = new HBaseImport("-t", dbNames[0] + "." + tableNames[1]);
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    // Make sure there aren't any extra roles
-    Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
-    Database db = store.getDatabase(dbNames[0]);
-    Assert.assertNotNull(db);
-
-    Table table = store.getTable(db.getName(), tableNames[1]);
-    Assert.assertNotNull(table);
-    Assert.assertEquals(1, store.getAllTables(db.getName()).size());
-
-    for (int j = 0; j < partVals.length; j++) {
-      Partition part = store.getPartition(dbNames[0], tableNames[1], Arrays.asList(partVals[j]));
-      Assert.assertNotNull(part);
-      Assert.assertEquals("file:/tmp/region=" + partVals[j], part.getSd().getLocation());
-    }
-    Assert.assertEquals(4, store.getPartitions(dbNames[0], tableNames[1], -1).size());
-
-    Assert.assertNull(store.getTable(db.getName(), tableNames[0]));
-
-    List<Index> indexes = store.getIndexes(db.getName(), tableNames[1], -1);
-    Assert.assertEquals(0, indexes.size());
-
-    Assert.assertEquals(0, store.getFunctions(dbNames[0], "*").size());
-    Assert.assertEquals(baseNumDbs + 1, store.getAllDatabases().size());
-
-    Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
-    String[] hbaseKeys = store.getMasterKeys();
-    Assert.assertEquals(baseNumKeys, hbaseKeys.length);
-  }
-
-  @Test
-  public void importSecurity() throws Exception {
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"securitydb1", "securitydb2"};
-    String[] roles = new String[] {"securityrole1", "securityrole2"};
-    String[] tokenIds = new String[] {"securitytokenid1", "securitytokenid2"};
-    String[] tokens = new String[] {"securitytoken1", "securitytoken2"};
-    String[] masterKeys = new String[] {"securitymk1", "securitymk2"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
-    int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
-    int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
-
-    HBaseImport importer = new HBaseImport("-k");
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
-    Assert.assertEquals(baseNumDbs, store.getAllDatabases().size());
-
-    // I can't test total number of tokens or master keys because the import grabs all and copies
-    // them, which means it grabs the ones imported by importAll test (if it's already run).
-    // Depending on it already running would make the tests order dependent, which junit doesn't
-    // guarantee.
-    for (int i = 0; i < tokenIds.length; i++) {
-      Assert.assertEquals(tokens[i], store.getToken(tokenIds[i]));
-    }
-    String[] hbaseKeys = store.getMasterKeys();
-    Set<String> keys = new HashSet<>(Arrays.asList(hbaseKeys));
-    for (int i = 0; i < masterKeys.length; i++) {
-      Assert.assertTrue(keys.contains(masterKeys[i]));
-    }
-  }
-
-  // TODO test for bogus function name
-  // TODO test for bogus table name
-  // TODO test for non-existent items
-
-  @Test
-  public void importOneRole() throws Exception {
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"oneroledb1", "oneroledb2"};
-    String[] roles = new String[] {"onerolerole1", "onerolerole2"};
-    String[] tokenIds = new String[] {"oneroletokenid1", "oneroletokenid2"};
-    String[] tokens = new String[] {"oneroletoken1", "oneroletoken2"};
-    String[] masterKeys = new String[] {"onerolemk1", "onerolemk2"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
-    int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
-    int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
-    int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
-        store.getAllTokenIdentifiers().size();
-    int baseNumKeys =  store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
-    HBaseImport importer = new HBaseImport("-r", roles[0]);
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    Role role = store.getRole(roles[0]);
-    Assert.assertNotNull(role);
-    Assert.assertEquals(roles[0], role.getRoleName());
-
-    // Make sure there aren't any extra roles
-    Assert.assertEquals(baseNumRoles + 1, store.listRoleNames().size());
-    Assert.assertEquals(baseNumDbs, store.getAllDatabases().size());
-
-    Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
-    String[] hbaseKeys = store.getMasterKeys();
-    Assert.assertEquals(baseNumKeys, hbaseKeys.length);
-
-    // Have to do this last as it will throw an exception
-    thrown.expect(NoSuchObjectException.class);
-    store.getRole(roles[1]);
-  }
-
-  private void setupObjectStore(RawStore rdbms, String[] roles, String[] dbNames,
-                                String[] tokenIds, String[] tokens, String[] masterKeys, int now)
-      throws MetaException, InvalidObjectException, NoSuchObjectException {
-    setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now, false);
-  }
-
-  private void setupObjectStore(RawStore rdbms, String[] dbNames, int now,
-                                boolean putConstraintsOnTables)
-      throws MetaException, InvalidObjectException, NoSuchObjectException {
-    setupObjectStore(rdbms, null, dbNames, null, null, null, now, putConstraintsOnTables);
-  }
-
-  private void setupObjectStore(RawStore rdbms, String[] roles, String[] dbNames,
-                                String[] tokenIds, String[] tokens, String[] masterKeys, int now,
-                                boolean putConstraintsOnTables)
-      throws MetaException, InvalidObjectException, NoSuchObjectException {
-    if (roles != null) {
-      for (int i = 0; i < roles.length; i++) {
-        rdbms.addRole(roles[i], "me");
-      }
-    }
-
-    for (int i = 0; i < dbNames.length; i++) {
-      rdbms.createDatabase(
-          new Database(dbNames[i], "no description", "file:/tmp", emptyParameters));
-
-      List<FieldSchema> cols = new ArrayList<>();
-      cols.add(new FieldSchema("col1", "int", "nocomment"));
-      SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-      StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-          serde, null, null, emptyParameters);
-      rdbms.createTable(new Table(tableNames[0], dbNames[i], "me", now, now, 0, sd, null,
-          emptyParameters, null, null, null));
-      if (putConstraintsOnTables) {
-        rdbms.addPrimaryKeys(Collections.singletonList(
-            new SQLPrimaryKey(dbNames[i], tableNames[0], "col1", 0, dbNames[i] + "_" + pkNames[0],
-                true, false, true)
-        ));
-      }
-
-      List<FieldSchema> partCols = new ArrayList<>();
-      partCols.add(new FieldSchema("region", "string", ""));
-      rdbms.createTable(new Table(tableNames[1], dbNames[i], "me", now, now, 0, sd, partCols,
-          emptyParameters, null, null, null));
-      if (putConstraintsOnTables) {
-        rdbms.addPrimaryKeys(Arrays.asList(
-            new SQLPrimaryKey(dbNames[i], tableNames[1], "col1", 0, dbNames[i] + "_" + pkNames[1],
-                true, false, true)
-        ));
-        rdbms.addForeignKeys(Collections.singletonList(
-            new SQLForeignKey(dbNames[i], tableNames[0], "col1", dbNames[i], tableNames[1],
-                "col1", 0, 1, 2, dbNames[i] + "_" + fkNames[1], dbNames[i] + "_" + pkNames[0],
-                true, false, true)
-        ));
-
-      }
-
-      for (int j = 0; j < partVals.length; j++) {
-        StorageDescriptor psd = new StorageDescriptor(sd);
-        psd.setLocation("file:/tmp/region=" + partVals[j]);
-        Partition part = new Partition(Arrays.asList(partVals[j]), dbNames[i], tableNames[1],
-            now, now, psd, emptyParameters);
-        rdbms.addPartition(part);
-      }
-
-      for (String funcName : funcNames) {
-        LOG.debug("Creating new function " + dbNames[i] + "." + funcName);
-        rdbms.createFunction(new Function(funcName, dbNames[i], "classname", "ownername",
-            PrincipalType.USER, (int) System.currentTimeMillis() / 1000, FunctionType.JAVA,
-            Arrays.asList(new ResourceUri(ResourceType.JAR, "uri"))));
-      }
-
-      for (String indexName : indexNames) {
-        LOG.debug("Creating new index " + dbNames[i] + "." + tableNames[0] + "." + indexName);
-        String indexTableName = tableNames[0] + "__" + indexName + "__";
-        rdbms.createTable(new Table(indexTableName, dbNames[i], "me", now, now, 0, sd, partCols,
-            emptyParameters, null, null, null));
-        rdbms.addIndex(new Index(indexName, null, dbNames[i], tableNames[0],
-            now, now, indexTableName, sd, emptyParameters, false));
-      }
-    }
-    if (tokenIds != null) {
-      for (int i = 0; i < tokenIds.length; i++) rdbms.addToken(tokenIds[i], tokens[i]);
-    }
-    if (masterKeys != null) {
-      for (int i = 0; i < masterKeys.length; i++) {
-        masterKeySeqs.add(rdbms.addMasterKey(masterKeys[i]));
-      }
-    }
-  }
-
-  @Test
-  public void parallel() throws Exception {
-    int parallelFactor = 10;
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"paralleldb1"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    for (int i = 0; i < dbNames.length; i++) {
-      rdbms.createDatabase(
-          new Database(dbNames[i], "no description", "file:/tmp", emptyParameters));
-
-      List<FieldSchema> cols = new ArrayList<>();
-      cols.add(new FieldSchema("col1", "int", "nocomment"));
-      SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-      StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-          serde, null, null, emptyParameters);
-
-      List<FieldSchema> partCols = new ArrayList<>();
-      partCols.add(new FieldSchema("region", "string", ""));
-      for (int j = 0; j < parallelFactor; j++) {
-        rdbms.createTable(new Table("t" + j, dbNames[i], "me", now, now, 0, sd, partCols,
-            emptyParameters, null, null, null));
-        for (int k = 0; k < parallelFactor; k++) {
-          StorageDescriptor psd = new StorageDescriptor(sd);
-          psd.setLocation("file:/tmp/region=" + k);
-          Partition part = new Partition(Arrays.asList("p" + k), dbNames[i], "t" + j,
-              now, now, psd, emptyParameters);
-          rdbms.addPartition(part);
-        }
-      }
-    }
-
-    HBaseImport importer = new HBaseImport("-p", "2", "-b", "2", "-d", dbNames[0]);
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    for (int i = 0; i < dbNames.length; i++) {
-      Database db = store.getDatabase(dbNames[i]);
-      Assert.assertNotNull(db);
-
-      for (int j = 0; j < parallelFactor; j++) {
-        Table table = store.getTable(db.getName(), "t" + j);
-        Assert.assertNotNull(table);
-        Assert.assertEquals(now, table.getLastAccessTime());
-        Assert.assertEquals("input", table.getSd().getInputFormat());
-
-        for (int k = 0; k < parallelFactor; k++) {
-          Partition part =
-              store.getPartition(dbNames[i], "t" + j, Arrays.asList("p" + k));
-          Assert.assertNotNull(part);
-          Assert.assertEquals("file:/tmp/region=" + k, part.getSd().getLocation());
-        }
-
-        Assert.assertEquals(parallelFactor, store.getPartitions(dbNames[i], "t" + j, -1).size());
-      }
-      Assert.assertEquals(parallelFactor, store.getAllTables(dbNames[i]).size());
-
-    }
-  }
-
-  // Same as the test above except we create 9 of everything instead of 10.  This is important
-  // because in using a batch size of 2 the previous test guarantees 10 /2 =5 , meaning we'll
-  // have 5 writes on the partition queue with exactly 2 entries.  In this test we'll handle the
-  // case where the last entry in the queue has fewer partitions.
-  @Test
-  public void parallelOdd() throws Exception {
-    int parallelFactor = 9;
-    RawStore rdbms;
-    rdbms = new ObjectStore();
-    rdbms.setConf(conf);
-
-    String[] dbNames = new String[] {"oddparalleldb1"};
-    int now = (int)System.currentTimeMillis() / 1000;
-
-    for (int i = 0; i < dbNames.length; i++) {
-      rdbms.createDatabase(
-          new Database(dbNames[i], "no description", "file:/tmp", emptyParameters));
-
-      List<FieldSchema> cols = new ArrayList<>();
-      cols.add(new FieldSchema("col1", "int", "nocomment"));
-      SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
-      StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
-          serde, null, null, emptyParameters);
-
-      List<FieldSchema> partCols = new ArrayList<>();
-      partCols.add(new FieldSchema("region", "string", ""));
-      for (int j = 0; j < parallelFactor; j++) {
-        rdbms.createTable(new Table("t" + j, dbNames[i], "me", now, now, 0, sd, partCols,
-            emptyParameters, null, null, null));
-        for (int k = 0; k < parallelFactor; k++) {
-          StorageDescriptor psd = new StorageDescriptor(sd);
-          psd.setLocation("file:/tmp/region=" + k);
-          Partition part = new Partition(Arrays.asList("p" + k), dbNames[i], "t" + j,
-              now, now, psd, emptyParameters);
-          rdbms.addPartition(part);
-        }
-      }
-    }
-
-    HBaseImport importer = new HBaseImport("-p", "2", "-b", "2", "-d", dbNames[0]);
-    importer.setConnections(rdbms, store);
-    importer.run();
-
-    for (int i = 0; i < dbNames.length; i++) {
-      Database db = store.getDatabase(dbNames[i]);
-      Assert.assertNotNull(db);
-
-      for (int j = 0; j < parallelFactor; j++) {
-        Table table = store.getTable(db.getName(), "t" + j);
-        Assert.assertNotNull(table);
-        Assert.assertEquals(now, table.getLastAccessTime());
-        Assert.assertEquals("input", table.getSd().getInputFormat());
-
-        for (int k = 0; k < parallelFactor; k++) {
-          Partition part =
-              store.getPartition(dbNames[i], "t" + j, Arrays.asList("p" + k));
-          Assert.assertNotNull(part);
-          Assert.assertEquals("file:/tmp/region=" + k, part.getSd().getLocation());
-        }
-
-        Assert.assertEquals(parallelFactor, store.getPartitions(dbNames[i], "t" + j, -1).size());
-      }
-      Assert.assertEquals(parallelFactor, store.getAllTables(dbNames[i]).size());
-
-    }
-  }
-}


[13/17] hive git commit: HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/hbase/HbaseMetastoreProto.java
----------------------------------------------------------------------
diff --git a/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/hbase/HbaseMetastoreProto.java b/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/hbase/HbaseMetastoreProto.java
deleted file mode 100644
index 9cf1ee2..0000000
--- a/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/hbase/HbaseMetastoreProto.java
+++ /dev/null
@@ -1,46709 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: hbase_metastore_proto.proto
-
-package org.apache.hadoop.hive.metastore.hbase;
-
-public final class HbaseMetastoreProto {
-  private HbaseMetastoreProto() {}
-  public static void registerAllExtensions(
-      com.google.protobuf.ExtensionRegistry registry) {
-  }
-  /**
-   * Protobuf enum {@code org.apache.hadoop.hive.metastore.hbase.PrincipalType}
-   */
-  public enum PrincipalType
-      implements com.google.protobuf.ProtocolMessageEnum {
-    /**
-     * <code>USER = 0;</code>
-     */
-    USER(0, 0),
-    /**
-     * <code>ROLE = 1;</code>
-     */
-    ROLE(1, 1),
-    ;
-
-    /**
-     * <code>USER = 0;</code>
-     */
-    public static final int USER_VALUE = 0;
-    /**
-     * <code>ROLE = 1;</code>
-     */
-    public static final int ROLE_VALUE = 1;
-
-
-    public final int getNumber() { return value; }
-
-    public static PrincipalType valueOf(int value) {
-      switch (value) {
-        case 0: return USER;
-        case 1: return ROLE;
-        default: return null;
-      }
-    }
-
-    public static com.google.protobuf.Internal.EnumLiteMap<PrincipalType>
-        internalGetValueMap() {
-      return internalValueMap;
-    }
-    private static com.google.protobuf.Internal.EnumLiteMap<PrincipalType>
-        internalValueMap =
-          new com.google.protobuf.Internal.EnumLiteMap<PrincipalType>() {
-            public PrincipalType findValueByNumber(int number) {
-              return PrincipalType.valueOf(number);
-            }
-          };
-
-    public final com.google.protobuf.Descriptors.EnumValueDescriptor
-        getValueDescriptor() {
-      return getDescriptor().getValues().get(index);
-    }
-    public final com.google.protobuf.Descriptors.EnumDescriptor
-        getDescriptorForType() {
-      return getDescriptor();
-    }
-    public static final com.google.protobuf.Descriptors.EnumDescriptor
-        getDescriptor() {
-      return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.getDescriptor().getEnumTypes().get(0);
-    }
-
-    private static final PrincipalType[] VALUES = values();
-
-    public static PrincipalType valueOf(
-        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
-      if (desc.getType() != getDescriptor()) {
-        throw new java.lang.IllegalArgumentException(
-          "EnumValueDescriptor is not for this type.");
-      }
-      return VALUES[desc.getIndex()];
-    }
-
-    private final int index;
-    private final int value;
-
-    private PrincipalType(int index, int value) {
-      this.index = index;
-      this.value = value;
-    }
-
-    // @@protoc_insertion_point(enum_scope:org.apache.hadoop.hive.metastore.hbase.PrincipalType)
-  }
-
-  public interface AggrStatsOrBuilder
-      extends com.google.protobuf.MessageOrBuilder {
-
-    // required int64 parts_found = 1;
-    /**
-     * <code>required int64 parts_found = 1;</code>
-     */
-    boolean hasPartsFound();
-    /**
-     * <code>required int64 parts_found = 1;</code>
-     */
-    long getPartsFound();
-
-    // repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> 
-        getColStatsList();
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats getColStats(int index);
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    int getColStatsCount();
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    java.util.List<? extends org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder> 
-        getColStatsOrBuilderList();
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder getColStatsOrBuilder(
-        int index);
-  }
-  /**
-   * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStats}
-   */
-  public static final class AggrStats extends
-      com.google.protobuf.GeneratedMessage
-      implements AggrStatsOrBuilder {
-    // Use AggrStats.newBuilder() to construct.
-    private AggrStats(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
-      super(builder);
-      this.unknownFields = builder.getUnknownFields();
-    }
-    private AggrStats(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
-    private static final AggrStats defaultInstance;
-    public static AggrStats getDefaultInstance() {
-      return defaultInstance;
-    }
-
-    public AggrStats getDefaultInstanceForType() {
-      return defaultInstance;
-    }
-
-    private final com.google.protobuf.UnknownFieldSet unknownFields;
-    @java.lang.Override
-    public final com.google.protobuf.UnknownFieldSet
-        getUnknownFields() {
-      return this.unknownFields;
-    }
-    private AggrStats(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      initFields();
-      int mutable_bitField0_ = 0;
-      com.google.protobuf.UnknownFieldSet.Builder unknownFields =
-          com.google.protobuf.UnknownFieldSet.newBuilder();
-      try {
-        boolean done = false;
-        while (!done) {
-          int tag = input.readTag();
-          switch (tag) {
-            case 0:
-              done = true;
-              break;
-            default: {
-              if (!parseUnknownField(input, unknownFields,
-                                     extensionRegistry, tag)) {
-                done = true;
-              }
-              break;
-            }
-            case 8: {
-              bitField0_ |= 0x00000001;
-              partsFound_ = input.readInt64();
-              break;
-            }
-            case 18: {
-              if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
-                colStats_ = new java.util.ArrayList<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats>();
-                mutable_bitField0_ |= 0x00000002;
-              }
-              colStats_.add(input.readMessage(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.PARSER, extensionRegistry));
-              break;
-            }
-          }
-        }
-      } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-        throw e.setUnfinishedMessage(this);
-      } catch (java.io.IOException e) {
-        throw new com.google.protobuf.InvalidProtocolBufferException(
-            e.getMessage()).setUnfinishedMessage(this);
-      } finally {
-        if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
-          colStats_ = java.util.Collections.unmodifiableList(colStats_);
-        }
-        this.unknownFields = unknownFields.build();
-        makeExtensionsImmutable();
-      }
-    }
-    public static final com.google.protobuf.Descriptors.Descriptor
-        getDescriptor() {
-      return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_descriptor;
-    }
-
-    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-        internalGetFieldAccessorTable() {
-      return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_fieldAccessorTable
-          .ensureFieldAccessorsInitialized(
-              org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.Builder.class);
-    }
-
-    public static com.google.protobuf.Parser<AggrStats> PARSER =
-        new com.google.protobuf.AbstractParser<AggrStats>() {
-      public AggrStats parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new AggrStats(input, extensionRegistry);
-      }
-    };
-
-    @java.lang.Override
-    public com.google.protobuf.Parser<AggrStats> getParserForType() {
-      return PARSER;
-    }
-
-    private int bitField0_;
-    // required int64 parts_found = 1;
-    public static final int PARTS_FOUND_FIELD_NUMBER = 1;
-    private long partsFound_;
-    /**
-     * <code>required int64 parts_found = 1;</code>
-     */
-    public boolean hasPartsFound() {
-      return ((bitField0_ & 0x00000001) == 0x00000001);
-    }
-    /**
-     * <code>required int64 parts_found = 1;</code>
-     */
-    public long getPartsFound() {
-      return partsFound_;
-    }
-
-    // repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;
-    public static final int COL_STATS_FIELD_NUMBER = 2;
-    private java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> colStats_;
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    public java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> getColStatsList() {
-      return colStats_;
-    }
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    public java.util.List<? extends org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder> 
-        getColStatsOrBuilderList() {
-      return colStats_;
-    }
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    public int getColStatsCount() {
-      return colStats_.size();
-    }
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats getColStats(int index) {
-      return colStats_.get(index);
-    }
-    /**
-     * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-     */
-    public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder getColStatsOrBuilder(
-        int index) {
-      return colStats_.get(index);
-    }
-
-    private void initFields() {
-      partsFound_ = 0L;
-      colStats_ = java.util.Collections.emptyList();
-    }
-    private byte memoizedIsInitialized = -1;
-    public final boolean isInitialized() {
-      byte isInitialized = memoizedIsInitialized;
-      if (isInitialized != -1) return isInitialized == 1;
-
-      if (!hasPartsFound()) {
-        memoizedIsInitialized = 0;
-        return false;
-      }
-      for (int i = 0; i < getColStatsCount(); i++) {
-        if (!getColStats(i).isInitialized()) {
-          memoizedIsInitialized = 0;
-          return false;
-        }
-      }
-      memoizedIsInitialized = 1;
-      return true;
-    }
-
-    public void writeTo(com.google.protobuf.CodedOutputStream output)
-                        throws java.io.IOException {
-      getSerializedSize();
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        output.writeInt64(1, partsFound_);
-      }
-      for (int i = 0; i < colStats_.size(); i++) {
-        output.writeMessage(2, colStats_.get(i));
-      }
-      getUnknownFields().writeTo(output);
-    }
-
-    private int memoizedSerializedSize = -1;
-    public int getSerializedSize() {
-      int size = memoizedSerializedSize;
-      if (size != -1) return size;
-
-      size = 0;
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeInt64Size(1, partsFound_);
-      }
-      for (int i = 0; i < colStats_.size(); i++) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeMessageSize(2, colStats_.get(i));
-      }
-      size += getUnknownFields().getSerializedSize();
-      memoizedSerializedSize = size;
-      return size;
-    }
-
-    private static final long serialVersionUID = 0L;
-    @java.lang.Override
-    protected java.lang.Object writeReplace()
-        throws java.io.ObjectStreamException {
-      return super.writeReplace();
-    }
-
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
-        com.google.protobuf.ByteString data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
-        com.google.protobuf.ByteString data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(byte[] data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
-        byte[] data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseDelimitedFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseDelimitedFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
-        com.google.protobuf.CodedInputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-
-    public static Builder newBuilder() { return Builder.create(); }
-    public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats prototype) {
-      return newBuilder().mergeFrom(prototype);
-    }
-    public Builder toBuilder() { return newBuilder(this); }
-
-    @java.lang.Override
-    protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-      Builder builder = new Builder(parent);
-      return builder;
-    }
-    /**
-     * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStats}
-     */
-    public static final class Builder extends
-        com.google.protobuf.GeneratedMessage.Builder<Builder>
-       implements org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsOrBuilder {
-      public static final com.google.protobuf.Descriptors.Descriptor
-          getDescriptor() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_descriptor;
-      }
-
-      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-          internalGetFieldAccessorTable() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_fieldAccessorTable
-            .ensureFieldAccessorsInitialized(
-                org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.Builder.class);
-      }
-
-      // Construct using org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.newBuilder()
-      private Builder() {
-        maybeForceBuilderInitialization();
-      }
-
-      private Builder(
-          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-        super(parent);
-        maybeForceBuilderInitialization();
-      }
-      private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
-          getColStatsFieldBuilder();
-        }
-      }
-      private static Builder create() {
-        return new Builder();
-      }
-
-      public Builder clear() {
-        super.clear();
-        partsFound_ = 0L;
-        bitField0_ = (bitField0_ & ~0x00000001);
-        if (colStatsBuilder_ == null) {
-          colStats_ = java.util.Collections.emptyList();
-          bitField0_ = (bitField0_ & ~0x00000002);
-        } else {
-          colStatsBuilder_.clear();
-        }
-        return this;
-      }
-
-      public Builder clone() {
-        return create().mergeFrom(buildPartial());
-      }
-
-      public com.google.protobuf.Descriptors.Descriptor
-          getDescriptorForType() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_descriptor;
-      }
-
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats getDefaultInstanceForType() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.getDefaultInstance();
-      }
-
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats build() {
-        org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats result = buildPartial();
-        if (!result.isInitialized()) {
-          throw newUninitializedMessageException(result);
-        }
-        return result;
-      }
-
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats buildPartial() {
-        org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats result = new org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats(this);
-        int from_bitField0_ = bitField0_;
-        int to_bitField0_ = 0;
-        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
-          to_bitField0_ |= 0x00000001;
-        }
-        result.partsFound_ = partsFound_;
-        if (colStatsBuilder_ == null) {
-          if (((bitField0_ & 0x00000002) == 0x00000002)) {
-            colStats_ = java.util.Collections.unmodifiableList(colStats_);
-            bitField0_ = (bitField0_ & ~0x00000002);
-          }
-          result.colStats_ = colStats_;
-        } else {
-          result.colStats_ = colStatsBuilder_.build();
-        }
-        result.bitField0_ = to_bitField0_;
-        onBuilt();
-        return result;
-      }
-
-      public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats) {
-          return mergeFrom((org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats)other);
-        } else {
-          super.mergeFrom(other);
-          return this;
-        }
-      }
-
-      public Builder mergeFrom(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats other) {
-        if (other == org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.getDefaultInstance()) return this;
-        if (other.hasPartsFound()) {
-          setPartsFound(other.getPartsFound());
-        }
-        if (colStatsBuilder_ == null) {
-          if (!other.colStats_.isEmpty()) {
-            if (colStats_.isEmpty()) {
-              colStats_ = other.colStats_;
-              bitField0_ = (bitField0_ & ~0x00000002);
-            } else {
-              ensureColStatsIsMutable();
-              colStats_.addAll(other.colStats_);
-            }
-            onChanged();
-          }
-        } else {
-          if (!other.colStats_.isEmpty()) {
-            if (colStatsBuilder_.isEmpty()) {
-              colStatsBuilder_.dispose();
-              colStatsBuilder_ = null;
-              colStats_ = other.colStats_;
-              bitField0_ = (bitField0_ & ~0x00000002);
-              colStatsBuilder_ = 
-                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
-                   getColStatsFieldBuilder() : null;
-            } else {
-              colStatsBuilder_.addAllMessages(other.colStats_);
-            }
-          }
-        }
-        this.mergeUnknownFields(other.getUnknownFields());
-        return this;
-      }
-
-      public final boolean isInitialized() {
-        if (!hasPartsFound()) {
-          
-          return false;
-        }
-        for (int i = 0; i < getColStatsCount(); i++) {
-          if (!getColStats(i).isInitialized()) {
-            
-            return false;
-          }
-        }
-        return true;
-      }
-
-      public Builder mergeFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws java.io.IOException {
-        org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parsedMessage = null;
-        try {
-          parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
-        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats) e.getUnfinishedMessage();
-          throw e;
-        } finally {
-          if (parsedMessage != null) {
-            mergeFrom(parsedMessage);
-          }
-        }
-        return this;
-      }
-      private int bitField0_;
-
-      // required int64 parts_found = 1;
-      private long partsFound_ ;
-      /**
-       * <code>required int64 parts_found = 1;</code>
-       */
-      public boolean hasPartsFound() {
-        return ((bitField0_ & 0x00000001) == 0x00000001);
-      }
-      /**
-       * <code>required int64 parts_found = 1;</code>
-       */
-      public long getPartsFound() {
-        return partsFound_;
-      }
-      /**
-       * <code>required int64 parts_found = 1;</code>
-       */
-      public Builder setPartsFound(long value) {
-        bitField0_ |= 0x00000001;
-        partsFound_ = value;
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>required int64 parts_found = 1;</code>
-       */
-      public Builder clearPartsFound() {
-        bitField0_ = (bitField0_ & ~0x00000001);
-        partsFound_ = 0L;
-        onChanged();
-        return this;
-      }
-
-      // repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;
-      private java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> colStats_ =
-        java.util.Collections.emptyList();
-      private void ensureColStatsIsMutable() {
-        if (!((bitField0_ & 0x00000002) == 0x00000002)) {
-          colStats_ = new java.util.ArrayList<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats>(colStats_);
-          bitField0_ |= 0x00000002;
-         }
-      }
-
-      private com.google.protobuf.RepeatedFieldBuilder<
-          org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder> colStatsBuilder_;
-
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> getColStatsList() {
-        if (colStatsBuilder_ == null) {
-          return java.util.Collections.unmodifiableList(colStats_);
-        } else {
-          return colStatsBuilder_.getMessageList();
-        }
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public int getColStatsCount() {
-        if (colStatsBuilder_ == null) {
-          return colStats_.size();
-        } else {
-          return colStatsBuilder_.getCount();
-        }
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats getColStats(int index) {
-        if (colStatsBuilder_ == null) {
-          return colStats_.get(index);
-        } else {
-          return colStatsBuilder_.getMessage(index);
-        }
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public Builder setColStats(
-          int index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats value) {
-        if (colStatsBuilder_ == null) {
-          if (value == null) {
-            throw new NullPointerException();
-          }
-          ensureColStatsIsMutable();
-          colStats_.set(index, value);
-          onChanged();
-        } else {
-          colStatsBuilder_.setMessage(index, value);
-        }
-        return this;
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public Builder setColStats(
-          int index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder builderForValue) {
-        if (colStatsBuilder_ == null) {
-          ensureColStatsIsMutable();
-          colStats_.set(index, builderForValue.build());
-          onChanged();
-        } else {
-          colStatsBuilder_.setMessage(index, builderForValue.build());
-        }
-        return this;
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public Builder addColStats(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats value) {
-        if (colStatsBuilder_ == null) {
-          if (value == null) {
-            throw new NullPointerException();
-          }
-          ensureColStatsIsMutable();
-          colStats_.add(value);
-          onChanged();
-        } else {
-          colStatsBuilder_.addMessage(value);
-        }
-        return this;
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public Builder addColStats(
-          int index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats value) {
-        if (colStatsBuilder_ == null) {
-          if (value == null) {
-            throw new NullPointerException();
-          }
-          ensureColStatsIsMutable();
-          colStats_.add(index, value);
-          onChanged();
-        } else {
-          colStatsBuilder_.addMessage(index, value);
-        }
-        return this;
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public Builder addColStats(
-          org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder builderForValue) {
-        if (colStatsBuilder_ == null) {
-          ensureColStatsIsMutable();
-          colStats_.add(builderForValue.build());
-          onChanged();
-        } else {
-          colStatsBuilder_.addMessage(builderForValue.build());
-        }
-        return this;
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public Builder addColStats(
-          int index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder builderForValue) {
-        if (colStatsBuilder_ == null) {
-          ensureColStatsIsMutable();
-          colStats_.add(index, builderForValue.build());
-          onChanged();
-        } else {
-          colStatsBuilder_.addMessage(index, builderForValue.build());
-        }
-        return this;
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public Builder addAllColStats(
-          java.lang.Iterable<? extends org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> values) {
-        if (colStatsBuilder_ == null) {
-          ensureColStatsIsMutable();
-          super.addAll(values, colStats_);
-          onChanged();
-        } else {
-          colStatsBuilder_.addAllMessages(values);
-        }
-        return this;
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public Builder clearColStats() {
-        if (colStatsBuilder_ == null) {
-          colStats_ = java.util.Collections.emptyList();
-          bitField0_ = (bitField0_ & ~0x00000002);
-          onChanged();
-        } else {
-          colStatsBuilder_.clear();
-        }
-        return this;
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public Builder removeColStats(int index) {
-        if (colStatsBuilder_ == null) {
-          ensureColStatsIsMutable();
-          colStats_.remove(index);
-          onChanged();
-        } else {
-          colStatsBuilder_.remove(index);
-        }
-        return this;
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder getColStatsBuilder(
-          int index) {
-        return getColStatsFieldBuilder().getBuilder(index);
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder getColStatsOrBuilder(
-          int index) {
-        if (colStatsBuilder_ == null) {
-          return colStats_.get(index);  } else {
-          return colStatsBuilder_.getMessageOrBuilder(index);
-        }
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public java.util.List<? extends org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder> 
-           getColStatsOrBuilderList() {
-        if (colStatsBuilder_ != null) {
-          return colStatsBuilder_.getMessageOrBuilderList();
-        } else {
-          return java.util.Collections.unmodifiableList(colStats_);
-        }
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder addColStatsBuilder() {
-        return getColStatsFieldBuilder().addBuilder(
-            org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.getDefaultInstance());
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder addColStatsBuilder(
-          int index) {
-        return getColStatsFieldBuilder().addBuilder(
-            index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.getDefaultInstance());
-      }
-      /**
-       * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
-       */
-      public java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder> 
-           getColStatsBuilderList() {
-        return getColStatsFieldBuilder().getBuilderList();
-      }
-      private com.google.protobuf.RepeatedFieldBuilder<
-          org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder> 
-          getColStatsFieldBuilder() {
-        if (colStatsBuilder_ == null) {
-          colStatsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
-              org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder>(
-                  colStats_,
-                  ((bitField0_ & 0x00000002) == 0x00000002),
-                  getParentForChildren(),
-                  isClean());
-          colStats_ = null;
-        }
-        return colStatsBuilder_;
-      }
-
-      // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.metastore.hbase.AggrStats)
-    }
-
-    static {
-      defaultInstance = new AggrStats(true);
-      defaultInstance.initFields();
-    }
-
-    // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.metastore.hbase.AggrStats)
-  }
-
-  public interface AggrStatsBloomFilterOrBuilder
-      extends com.google.protobuf.MessageOrBuilder {
-
-    // required bytes db_name = 1;
-    /**
-     * <code>required bytes db_name = 1;</code>
-     */
-    boolean hasDbName();
-    /**
-     * <code>required bytes db_name = 1;</code>
-     */
-    com.google.protobuf.ByteString getDbName();
-
-    // required bytes table_name = 2;
-    /**
-     * <code>required bytes table_name = 2;</code>
-     */
-    boolean hasTableName();
-    /**
-     * <code>required bytes table_name = 2;</code>
-     */
-    com.google.protobuf.ByteString getTableName();
-
-    // required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;
-    /**
-     * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-     */
-    boolean hasBloomFilter();
-    /**
-     * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-     */
-    org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter getBloomFilter();
-    /**
-     * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-     */
-    org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilterOrBuilder getBloomFilterOrBuilder();
-
-    // required int64 aggregated_at = 4;
-    /**
-     * <code>required int64 aggregated_at = 4;</code>
-     */
-    boolean hasAggregatedAt();
-    /**
-     * <code>required int64 aggregated_at = 4;</code>
-     */
-    long getAggregatedAt();
-  }
-  /**
-   * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter}
-   */
-  public static final class AggrStatsBloomFilter extends
-      com.google.protobuf.GeneratedMessage
-      implements AggrStatsBloomFilterOrBuilder {
-    // Use AggrStatsBloomFilter.newBuilder() to construct.
-    private AggrStatsBloomFilter(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
-      super(builder);
-      this.unknownFields = builder.getUnknownFields();
-    }
-    private AggrStatsBloomFilter(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
-    private static final AggrStatsBloomFilter defaultInstance;
-    public static AggrStatsBloomFilter getDefaultInstance() {
-      return defaultInstance;
-    }
-
-    public AggrStatsBloomFilter getDefaultInstanceForType() {
-      return defaultInstance;
-    }
-
-    private final com.google.protobuf.UnknownFieldSet unknownFields;
-    @java.lang.Override
-    public final com.google.protobuf.UnknownFieldSet
-        getUnknownFields() {
-      return this.unknownFields;
-    }
-    private AggrStatsBloomFilter(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      initFields();
-      int mutable_bitField0_ = 0;
-      com.google.protobuf.UnknownFieldSet.Builder unknownFields =
-          com.google.protobuf.UnknownFieldSet.newBuilder();
-      try {
-        boolean done = false;
-        while (!done) {
-          int tag = input.readTag();
-          switch (tag) {
-            case 0:
-              done = true;
-              break;
-            default: {
-              if (!parseUnknownField(input, unknownFields,
-                                     extensionRegistry, tag)) {
-                done = true;
-              }
-              break;
-            }
-            case 10: {
-              bitField0_ |= 0x00000001;
-              dbName_ = input.readBytes();
-              break;
-            }
-            case 18: {
-              bitField0_ |= 0x00000002;
-              tableName_ = input.readBytes();
-              break;
-            }
-            case 26: {
-              org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder subBuilder = null;
-              if (((bitField0_ & 0x00000004) == 0x00000004)) {
-                subBuilder = bloomFilter_.toBuilder();
-              }
-              bloomFilter_ = input.readMessage(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.PARSER, extensionRegistry);
-              if (subBuilder != null) {
-                subBuilder.mergeFrom(bloomFilter_);
-                bloomFilter_ = subBuilder.buildPartial();
-              }
-              bitField0_ |= 0x00000004;
-              break;
-            }
-            case 32: {
-              bitField0_ |= 0x00000008;
-              aggregatedAt_ = input.readInt64();
-              break;
-            }
-          }
-        }
-      } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-        throw e.setUnfinishedMessage(this);
-      } catch (java.io.IOException e) {
-        throw new com.google.protobuf.InvalidProtocolBufferException(
-            e.getMessage()).setUnfinishedMessage(this);
-      } finally {
-        this.unknownFields = unknownFields.build();
-        makeExtensionsImmutable();
-      }
-    }
-    public static final com.google.protobuf.Descriptors.Descriptor
-        getDescriptor() {
-      return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_descriptor;
-    }
-
-    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-        internalGetFieldAccessorTable() {
-      return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_fieldAccessorTable
-          .ensureFieldAccessorsInitialized(
-              org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.Builder.class);
-    }
-
-    public static com.google.protobuf.Parser<AggrStatsBloomFilter> PARSER =
-        new com.google.protobuf.AbstractParser<AggrStatsBloomFilter>() {
-      public AggrStatsBloomFilter parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new AggrStatsBloomFilter(input, extensionRegistry);
-      }
-    };
-
-    @java.lang.Override
-    public com.google.protobuf.Parser<AggrStatsBloomFilter> getParserForType() {
-      return PARSER;
-    }
-
-    public interface BloomFilterOrBuilder
-        extends com.google.protobuf.MessageOrBuilder {
-
-      // required int32 num_bits = 1;
-      /**
-       * <code>required int32 num_bits = 1;</code>
-       */
-      boolean hasNumBits();
-      /**
-       * <code>required int32 num_bits = 1;</code>
-       */
-      int getNumBits();
-
-      // required int32 num_funcs = 2;
-      /**
-       * <code>required int32 num_funcs = 2;</code>
-       */
-      boolean hasNumFuncs();
-      /**
-       * <code>required int32 num_funcs = 2;</code>
-       */
-      int getNumFuncs();
-
-      // repeated int64 bits = 3;
-      /**
-       * <code>repeated int64 bits = 3;</code>
-       */
-      java.util.List<java.lang.Long> getBitsList();
-      /**
-       * <code>repeated int64 bits = 3;</code>
-       */
-      int getBitsCount();
-      /**
-       * <code>repeated int64 bits = 3;</code>
-       */
-      long getBits(int index);
-    }
-    /**
-     * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter}
-     */
-    public static final class BloomFilter extends
-        com.google.protobuf.GeneratedMessage
-        implements BloomFilterOrBuilder {
-      // Use BloomFilter.newBuilder() to construct.
-      private BloomFilter(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
-        super(builder);
-        this.unknownFields = builder.getUnknownFields();
-      }
-      private BloomFilter(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
-      private static final BloomFilter defaultInstance;
-      public static BloomFilter getDefaultInstance() {
-        return defaultInstance;
-      }
-
-      public BloomFilter getDefaultInstanceForType() {
-        return defaultInstance;
-      }
-
-      private final com.google.protobuf.UnknownFieldSet unknownFields;
-      @java.lang.Override
-      public final com.google.protobuf.UnknownFieldSet
-          getUnknownFields() {
-        return this.unknownFields;
-      }
-      private BloomFilter(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        initFields();
-        int mutable_bitField0_ = 0;
-        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
-            com.google.protobuf.UnknownFieldSet.newBuilder();
-        try {
-          boolean done = false;
-          while (!done) {
-            int tag = input.readTag();
-            switch (tag) {
-              case 0:
-                done = true;
-                break;
-              default: {
-                if (!parseUnknownField(input, unknownFields,
-                                       extensionRegistry, tag)) {
-                  done = true;
-                }
-                break;
-              }
-              case 8: {
-                bitField0_ |= 0x00000001;
-                numBits_ = input.readInt32();
-                break;
-              }
-              case 16: {
-                bitField0_ |= 0x00000002;
-                numFuncs_ = input.readInt32();
-                break;
-              }
-              case 24: {
-                if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
-                  bits_ = new java.util.ArrayList<java.lang.Long>();
-                  mutable_bitField0_ |= 0x00000004;
-                }
-                bits_.add(input.readInt64());
-                break;
-              }
-              case 26: {
-                int length = input.readRawVarint32();
-                int limit = input.pushLimit(length);
-                if (!((mutable_bitField0_ & 0x00000004) == 0x00000004) && input.getBytesUntilLimit() > 0) {
-                  bits_ = new java.util.ArrayList<java.lang.Long>();
-                  mutable_bitField0_ |= 0x00000004;
-                }
-                while (input.getBytesUntilLimit() > 0) {
-                  bits_.add(input.readInt64());
-                }
-                input.popLimit(limit);
-                break;
-              }
-            }
-          }
-        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          throw e.setUnfinishedMessage(this);
-        } catch (java.io.IOException e) {
-          throw new com.google.protobuf.InvalidProtocolBufferException(
-              e.getMessage()).setUnfinishedMessage(this);
-        } finally {
-          if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
-            bits_ = java.util.Collections.unmodifiableList(bits_);
-          }
-          this.unknownFields = unknownFields.build();
-          makeExtensionsImmutable();
-        }
-      }
-      public static final com.google.protobuf.Descriptors.Descriptor
-          getDescriptor() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_descriptor;
-      }
-
-      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-          internalGetFieldAccessorTable() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_fieldAccessorTable
-            .ensureFieldAccessorsInitialized(
-                org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder.class);
-      }
-
-      public static com.google.protobuf.Parser<BloomFilter> PARSER =
-          new com.google.protobuf.AbstractParser<BloomFilter>() {
-        public BloomFilter parsePartialFrom(
-            com.google.protobuf.CodedInputStream input,
-            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-            throws com.google.protobuf.InvalidProtocolBufferException {
-          return new BloomFilter(input, extensionRegistry);
-        }
-      };
-
-      @java.lang.Override
-      public com.google.protobuf.Parser<BloomFilter> getParserForType() {
-        return PARSER;
-      }
-
-      private int bitField0_;
-      // required int32 num_bits = 1;
-      public static final int NUM_BITS_FIELD_NUMBER = 1;
-      private int numBits_;
-      /**
-       * <code>required int32 num_bits = 1;</code>
-       */
-      public boolean hasNumBits() {
-        return ((bitField0_ & 0x00000001) == 0x00000001);
-      }
-      /**
-       * <code>required int32 num_bits = 1;</code>
-       */
-      public int getNumBits() {
-        return numBits_;
-      }
-
-      // required int32 num_funcs = 2;
-      public static final int NUM_FUNCS_FIELD_NUMBER = 2;
-      private int numFuncs_;
-      /**
-       * <code>required int32 num_funcs = 2;</code>
-       */
-      public boolean hasNumFuncs() {
-        return ((bitField0_ & 0x00000002) == 0x00000002);
-      }
-      /**
-       * <code>required int32 num_funcs = 2;</code>
-       */
-      public int getNumFuncs() {
-        return numFuncs_;
-      }
-
-      // repeated int64 bits = 3;
-      public static final int BITS_FIELD_NUMBER = 3;
-      private java.util.List<java.lang.Long> bits_;
-      /**
-       * <code>repeated int64 bits = 3;</code>
-       */
-      public java.util.List<java.lang.Long>
-          getBitsList() {
-        return bits_;
-      }
-      /**
-       * <code>repeated int64 bits = 3;</code>
-       */
-      public int getBitsCount() {
-        return bits_.size();
-      }
-      /**
-       * <code>repeated int64 bits = 3;</code>
-       */
-      public long getBits(int index) {
-        return bits_.get(index);
-      }
-
-      private void initFields() {
-        numBits_ = 0;
-        numFuncs_ = 0;
-        bits_ = java.util.Collections.emptyList();
-      }
-      private byte memoizedIsInitialized = -1;
-      public final boolean isInitialized() {
-        byte isInitialized = memoizedIsInitialized;
-        if (isInitialized != -1) return isInitialized == 1;
-
-        if (!hasNumBits()) {
-          memoizedIsInitialized = 0;
-          return false;
-        }
-        if (!hasNumFuncs()) {
-          memoizedIsInitialized = 0;
-          return false;
-        }
-        memoizedIsInitialized = 1;
-        return true;
-      }
-
-      public void writeTo(com.google.protobuf.CodedOutputStream output)
-                          throws java.io.IOException {
-        getSerializedSize();
-        if (((bitField0_ & 0x00000001) == 0x00000001)) {
-          output.writeInt32(1, numBits_);
-        }
-        if (((bitField0_ & 0x00000002) == 0x00000002)) {
-          output.writeInt32(2, numFuncs_);
-        }
-        for (int i = 0; i < bits_.size(); i++) {
-          output.writeInt64(3, bits_.get(i));
-        }
-        getUnknownFields().writeTo(output);
-      }
-
-      private int memoizedSerializedSize = -1;
-      public int getSerializedSize() {
-        int size = memoizedSerializedSize;
-        if (size != -1) return size;
-
-        size = 0;
-        if (((bitField0_ & 0x00000001) == 0x00000001)) {
-          size += com.google.protobuf.CodedOutputStream
-            .computeInt32Size(1, numBits_);
-        }
-        if (((bitField0_ & 0x00000002) == 0x00000002)) {
-          size += com.google.protobuf.CodedOutputStream
-            .computeInt32Size(2, numFuncs_);
-        }
-        {
-          int dataSize = 0;
-          for (int i = 0; i < bits_.size(); i++) {
-            dataSize += com.google.protobuf.CodedOutputStream
-              .computeInt64SizeNoTag(bits_.get(i));
-          }
-          size += dataSize;
-          size += 1 * getBitsList().size();
-        }
-        size += getUnknownFields().getSerializedSize();
-        memoizedSerializedSize = size;
-        return size;
-      }
-
-      private static final long serialVersionUID = 0L;
-      @java.lang.Override
-      protected java.lang.Object writeReplace()
-          throws java.io.ObjectStreamException {
-        return super.writeReplace();
-      }
-
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
-          com.google.protobuf.ByteString data)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return PARSER.parseFrom(data);
-      }
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
-          com.google.protobuf.ByteString data,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return PARSER.parseFrom(data, extensionRegistry);
-      }
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(byte[] data)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return PARSER.parseFrom(data);
-      }
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
-          byte[] data,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return PARSER.parseFrom(data, extensionRegistry);
-      }
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(java.io.InputStream input)
-          throws java.io.IOException {
-        return PARSER.parseFrom(input);
-      }
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
-          java.io.InputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws java.io.IOException {
-        return PARSER.parseFrom(input, extensionRegistry);
-      }
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseDelimitedFrom(java.io.InputStream input)
-          throws java.io.IOException {
-        return PARSER.parseDelimitedFrom(input);
-      }
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseDelimitedFrom(
-          java.io.InputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws java.io.IOException {
-        return PARSER.parseDelimitedFrom(input, extensionRegistry);
-      }
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
-          com.google.protobuf.CodedInputStream input)
-          throws java.io.IOException {
-        return PARSER.parseFrom(input);
-      }
-      public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws java.io.IOException {
-        return PARSER.parseFrom(input, extensionRegistry);
-      }
-
-      public static Builder newBuilder() { return Builder.create(); }
-      public Builder newBuilderForType() { return newBuilder(); }
-      public static Builder newBuilder(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter prototype) {
-        return newBuilder().mergeFrom(prototype);
-      }
-      public Builder toBuilder() { return newBuilder(this); }
-
-      @java.lang.Override
-      protected Builder newBuilderForType(
-          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-        Builder builder = new Builder(parent);
-        return builder;
-      }
-      /**
-       * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter}
-       */
-      public static final class Builder extends
-          com.google.protobuf.GeneratedMessage.Builder<Builder>
-         implements org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilterOrBuilder {
-        public static final com.google.protobuf.Descriptors.Descriptor
-            getDescriptor() {
-          return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_descriptor;
-        }
-
-        protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-            internalGetFieldAccessorTable() {
-          return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_fieldAccessorTable
-              .ensureFieldAccessorsInitialized(
-                  org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder.class);
-        }
-
-        // Construct using org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.newBuilder()
-        private Builder() {
-          maybeForceBuilderInitialization();
-        }
-
-        private Builder(
-            com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-          super(parent);
-          maybeForceBuilderInitialization();
-        }
-        private void maybeForceBuilderInitialization() {
-          if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
-          }
-        }
-        private static Builder create() {
-          return new Builder();
-        }
-
-        public Builder clear() {
-          super.clear();
-          numBits_ = 0;
-          bitField0_ = (bitField0_ & ~0x00000001);
-          numFuncs_ = 0;
-          bitField0_ = (bitField0_ & ~0x00000002);
-          bits_ = java.util.Collections.emptyList();
-          bitField0_ = (bitField0_ & ~0x00000004);
-          return this;
-        }
-
-        public Builder clone() {
-          return create().mergeFrom(buildPartial());
-        }
-
-        public com.google.protobuf.Descriptors.Descriptor
-            getDescriptorForType() {
-          return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_descriptor;
-        }
-
-        public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter getDefaultInstanceForType() {
-          return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
-        }
-
-        public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter build() {
-          org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter result = buildPartial();
-          if (!result.isInitialized()) {
-            throw newUninitializedMessageException(result);
-          }
-          return result;
-        }
-
-        public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter buildPartial() {
-          org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter result = new org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter(this);
-          int from_bitField0_ = bitField0_;
-          int to_bitField0_ = 0;
-          if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
-            to_bitField0_ |= 0x00000001;
-          }
-          result.numBits_ = numBits_;
-          if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
-            to_bitField0_ |= 0x00000002;
-          }
-          result.numFuncs_ = numFuncs_;
-          if (((bitField0_ & 0x00000004) == 0x00000004)) {
-            bits_ = java.util.Collections.unmodifiableList(bits_);
-            bitField0_ = (bitField0_ & ~0x00000004);
-          }
-          result.bits_ = bits_;
-          result.bitField0_ = to_bitField0_;
-          onBuilt();
-          return result;
-        }
-
-        public Builder mergeFrom(com.google.protobuf.Message other) {
-          if (other instanceof org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter) {
-            return mergeFrom((org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter)other);
-          } else {
-            super.mergeFrom(other);
-            return this;
-          }
-        }
-
-        public Builder mergeFrom(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter other) {
-          if (other == org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance()) return this;
-          if (other.hasNumBits()) {
-            setNumBits(other.getNumBits());
-          }
-          if (other.hasNumFuncs()) {
-            setNumFuncs(other.getNumFuncs());
-          }
-          if (!other.bits_.isEmpty()) {
-            if (bits_.isEmpty()) {
-              bits_ = other.bits_;
-              bitField0_ = (bitField0_ & ~0x00000004);
-            } else {
-              ensureBitsIsMutable();
-              bits_.addAll(other.bits_);
-            }
-            onChanged();
-          }
-          this.mergeUnknownFields(other.getUnknownFields());
-          return this;
-        }
-
-        public final boolean isInitialized() {
-          if (!hasNumBits()) {
-            
-            return false;
-          }
-          if (!hasNumFuncs()) {
-            
-            return false;
-          }
-          return true;
-        }
-
-        public Builder mergeFrom(
-            com.google.protobuf.CodedInputStream input,
-            com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-            throws java.io.IOException {
-          org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parsedMessage = null;
-          try {
-            parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
-          } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-            parsedMessage = (org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter) e.getUnfinishedMessage();
-            throw e;
-          } finally {
-            if (parsedMessage != null) {
-              mergeFrom(parsedMessage);
-            }
-          }
-          return this;
-        }
-        private int bitField0_;
-
-        // required int32 num_bits = 1;
-        private int numBits_ ;
-        /**
-         * <code>required int32 num_bits = 1;</code>
-         */
-        public boolean hasNumBits() {
-          return ((bitField0_ & 0x00000001) == 0x00000001);
-        }
-        /**
-         * <code>required int32 num_bits = 1;</code>
-         */
-        public int getNumBits() {
-          return numBits_;
-        }
-        /**
-         * <code>required int32 num_bits = 1;</code>
-         */
-        public Builder setNumBits(int value) {
-          bitField0_ |= 0x00000001;
-          numBits_ = value;
-          onChanged();
-          return this;
-        }
-        /**
-         * <code>required int32 num_bits = 1;</code>
-         */
-        public Builder clearNumBits() {
-          bitField0_ = (bitField0_ & ~0x00000001);
-          numBits_ = 0;
-          onChanged();
-          return this;
-        }
-
-        // required int32 num_funcs = 2;
-        private int numFuncs_ ;
-        /**
-         * <code>required int32 num_funcs = 2;</code>
-         */
-        public boolean hasNumFuncs() {
-          return ((bitField0_ & 0x00000002) == 0x00000002);
-        }
-        /**
-         * <code>required int32 num_funcs = 2;</code>
-         */
-        public int getNumFuncs() {
-          return numFuncs_;
-        }
-        /**
-         * <code>required int32 num_funcs = 2;</code>
-         */
-        public Builder setNumFuncs(int value) {
-          bitField0_ |= 0x00000002;
-          numFuncs_ = value;
-          onChanged();
-          return this;
-        }
-        /**
-         * <code>required int32 num_funcs = 2;</code>
-         */
-        public Builder clearNumFuncs() {
-          bitField0_ = (bitField0_ & ~0x00000002);
-          numFuncs_ = 0;
-          onChanged();
-          return this;
-        }
-
-        // repeated int64 bits = 3;
-        private java.util.List<java.lang.Long> bits_ = java.util.Collections.emptyList();
-        private void ensureBitsIsMutable() {
-          if (!((bitField0_ & 0x00000004) == 0x00000004)) {
-            bits_ = new java.util.ArrayList<java.lang.Long>(bits_);
-            bitField0_ |= 0x00000004;
-           }
-        }
-        /**
-         * <code>repeated int64 bits = 3;</code>
-         */
-        public java.util.List<java.lang.Long>
-            getBitsList() {
-          return java.util.Collections.unmodifiableList(bits_);
-        }
-        /**
-         * <code>repeated int64 bits = 3;</code>
-         */
-        public int getBitsCount() {
-          return bits_.size();
-        }
-        /**
-         * <code>repeated int64 bits = 3;</code>
-         */
-        public long getBits(int index) {
-          return bits_.get(index);
-        }
-        /**
-         * <code>repeated int64 bits = 3;</code>
-         */
-        public Builder setBits(
-            int index, long value) {
-          ensureBitsIsMutable();
-          bits_.set(index, value);
-          onChanged();
-          return this;
-        }
-        /**
-         * <code>repeated int64 bits = 3;</code>
-         */
-        public Builder addBits(long value) {
-          ensureBitsIsMutable();
-          bits_.add(value);
-          onChanged();
-          return this;
-        }
-        /**
-         * <code>repeated int64 bits = 3;</code>
-         */
-        public Builder addAllBits(
-            java.lang.Iterable<? extends java.lang.Long> values) {
-          ensureBitsIsMutable();
-          super.addAll(values, bits_);
-          onChanged();
-          return this;
-        }
-        /**
-         * <code>repeated int64 bits = 3;</code>
-         */
-        public Builder clearBits() {
-          bits_ = java.util.Collections.emptyList();
-          bitField0_ = (bitField0_ & ~0x00000004);
-          onChanged();
-          return this;
-        }
-
-        // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter)
-      }
-
-      static {
-        defaultInstance = new BloomFilter(true);
-        defaultInstance.initFields();
-      }
-
-      // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter)
-    }
-
-    private int bitField0_;
-    // required bytes db_name = 1;
-    public static final int DB_NAME_FIELD_NUMBER = 1;
-    private com.google.protobuf.ByteString dbName_;
-    /**
-     * <code>required bytes db_name = 1;</code>
-     */
-    public boolean hasDbName() {
-      return ((bitField0_ & 0x00000001) == 0x00000001);
-    }
-    /**
-     * <code>required bytes db_name = 1;</code>
-     */
-    public com.google.protobuf.ByteString getDbName() {
-      return dbName_;
-    }
-
-    // required bytes table_name = 2;
-    public static final int TABLE_NAME_FIELD_NUMBER = 2;
-    private com.google.protobuf.ByteString tableName_;
-    /**
-     * <code>required bytes table_name = 2;</code>
-     */
-    public boolean hasTableName() {
-      return ((bitField0_ & 0x00000002) == 0x00000002);
-    }
-    /**
-     * <code>required bytes table_name = 2;</code>
-     */
-    public com.google.protobuf.ByteString getTableName() {
-      return tableName_;
-    }
-
-    // required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;
-    public static final int BLOOM_FILTER_FIELD_NUMBER = 3;
-    private org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter bloomFilter_;
-    /**
-     * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-     */
-    public boolean hasBloomFilter() {
-      return ((bitField0_ & 0x00000004) == 0x00000004);
-    }
-    /**
-     * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-     */
-    public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter getBloomFilter() {
-      return bloomFilter_;
-    }
-    /**
-     * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-     */
-    public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilterOrBuilder getBloomFilterOrBuilder() {
-      return bloomFilter_;
-    }
-
-    // required int64 aggregated_at = 4;
-    public static final int AGGREGATED_AT_FIELD_NUMBER = 4;
-    private long aggregatedAt_;
-    /**
-     * <code>required int64 aggregated_at = 4;</code>
-     */
-    public boolean hasAggregatedAt() {
-      return ((bitField0_ & 0x00000008) == 0x00000008);
-    }
-    /**
-     * <code>required int64 aggregated_at = 4;</code>
-     */
-    public long getAggregatedAt() {
-      return aggregatedAt_;
-    }
-
-    private void initFields() {
-      dbName_ = com.google.protobuf.ByteString.EMPTY;
-      tableName_ = com.google.protobuf.ByteString.EMPTY;
-      bloomFilter_ = org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
-      aggregatedAt_ = 0L;
-    }
-    private byte memoizedIsInitialized = -1;
-    public final boolean isInitialized() {
-      byte isInitialized = memoizedIsInitialized;
-      if (isInitialized != -1) return isInitialized == 1;
-
-      if (!hasDbName()) {
-        memoizedIsInitialized = 0;
-        return false;
-      }
-      if (!hasTableName()) {
-        memoizedIsInitialized = 0;
-        return false;
-      }
-      if (!hasBloomFilter()) {
-        memoizedIsInitialized = 0;
-        return false;
-      }
-      if (!hasAggregatedAt()) {
-        memoizedIsInitialized = 0;
-        return false;
-      }
-      if (!getBloomFilter().isInitialized()) {
-        memoizedIsInitialized = 0;
-        return false;
-      }
-      memoizedIsInitialized = 1;
-      return true;
-    }
-
-    public void writeTo(com.google.protobuf.CodedOutputStream output)
-                        throws java.io.IOException {
-      getSerializedSize();
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        output.writeBytes(1, dbName_);
-      }
-      if (((bitField0_ & 0x00000002) == 0x00000002)) {
-        output.writeBytes(2, tableName_);
-      }
-      if (((bitField0_ & 0x00000004) == 0x00000004)) {
-        output.writeMessage(3, bloomFilter_);
-      }
-      if (((bitField0_ & 0x00000008) == 0x00000008)) {
-        output.writeInt64(4, aggregatedAt_);
-      }
-      getUnknownFields().writeTo(output);
-    }
-
-    private int memoizedSerializedSize = -1;
-    public int getSerializedSize() {
-      int size = memoizedSerializedSize;
-      if (size != -1) return size;
-
-      size = 0;
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeBytesSize(1, dbName_);
-      }
-      if (((bitField0_ & 0x00000002) == 0x00000002)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeBytesSize(2, tableName_);
-      }
-      if (((bitField0_ & 0x00000004) == 0x00000004)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeMessageSize(3, bloomFilter_);
-      }
-      if (((bitField0_ & 0x00000008) == 0x00000008)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeInt64Size(4, aggregatedAt_);
-      }
-      size += getUnknownFields().getSerializedSize();
-      memoizedSerializedSize = size;
-      return size;
-    }
-
-    private static final long serialVersionUID = 0L;
-    @java.lang.Override
-    protected java.lang.Object writeReplace()
-        throws java.io.ObjectStreamException {
-      return super.writeReplace();
-    }
-
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
-        com.google.protobuf.ByteString data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
-        com.google.protobuf.ByteString data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(byte[] data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
-        byte[] data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseDelimitedFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseDelimitedFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
-        com.google.protobuf.CodedInputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-
-    public static Builder newBuilder() { return Builder.create(); }
-    public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter prototype) {
-      return newBuilder().mergeFrom(prototype);
-    }
-    public Builder toBuilder() { return newBuilder(this); }
-
-    @java.lang.Override
-    protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-      Builder builder = new Builder(parent);
-      return builder;
-    }
-    /**
-     * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter}
-     */
-    public static final class Builder extends
-        com.google.protobuf.GeneratedMessage.Builder<Builder>
-       implements org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilterOrBuilder {
-      public static final com.google.protobuf.Descriptors.Descriptor
-          getDescriptor() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_descriptor;
-      }
-
-      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-          internalGetFieldAccessorTable() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_fieldAccessorTable
-            .ensureFieldAccessorsInitialized(
-                org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.Builder.class);
-      }
-
-      // Construct using org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.newBuilder()
-      private Builder() {
-        maybeForceBuilderInitialization();
-      }
-
-      private Builder(
-          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-        super(parent);
-        maybeForceBuilderInitialization();
-      }
-      private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
-          getBloomFilterFieldBuilder();
-        }
-      }
-      private static Builder create() {
-        return new Builder();
-      }
-
-      public Builder clear() {
-        super.clear();
-        dbName_ = com.google.protobuf.ByteString.EMPTY;
-        bitField0_ = (bitField0_ & ~0x00000001);
-        tableName_ = com.google.protobuf.ByteString.EMPTY;
-        bitField0_ = (bitField0_ & ~0x00000002);
-        if (bloomFilterBuilder_ == null) {
-          bloomFilter_ = org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
-        } else {
-          bloomFilterBuilder_.clear();
-        }
-        bitField0_ = (bitField0_ & ~0x00000004);
-        aggregatedAt_ = 0L;
-        bitField0_ = (bitField0_ & ~0x00000008);
-        return this;
-      }
-
-      public Builder clone() {
-        return create().mergeFrom(buildPartial());
-      }
-
-      public com.google.protobuf.Descriptors.Descriptor
-          getDescriptorForType() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_descriptor;
-      }
-
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter getDefaultInstanceForType() {
-        return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.getDefaultInstance();
-      }
-
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter build() {
-        org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter result = buildPartial();
-        if (!result.isInitialized()) {
-          throw newUninitializedMessageException(result);
-        }
-        return result;
-      }
-
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter buildPartial() {
-        org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter result = new org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter(this);
-        int from_bitField0_ = bitField0_;
-        int to_bitField0_ = 0;
-        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
-          to_bitField0_ |= 0x00000001;
-        }
-        result.dbName_ = dbName_;
-        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
-          to_bitField0_ |= 0x00000002;
-        }
-        result.tableName_ = tableName_;
-        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
-          to_bitField0_ |= 0x00000004;
-        }
-        if (bloomFilterBuilder_ == null) {
-          result.bloomFilter_ = bloomFilter_;
-        } else {
-          result.bloomFilter_ = bloomFilterBuilder_.build();
-        }
-        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
-          to_bitField0_ |= 0x00000008;
-        }
-        result.aggregatedAt_ = aggregatedAt_;
-        result.bitField0_ = to_bitField0_;
-        onBuilt();
-        return result;
-      }
-
-      public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter) {
-          return mergeFrom((org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter)other);
-        } else {
-          super.mergeFrom(other);
-          return this;
-        }
-      }
-
-      public Builder mergeFrom(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter other) {
-        if (other == org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.getDefaultInstance()) return this;
-        if (other.hasDbName()) {
-          setDbName(other.getDbName());
-        }
-        if (other.hasTableName()) {
-          setTableName(other.getTableName());
-        }
-        if (other.hasBloomFilter()) {
-          mergeBloomFilter(other.getBloomFilter());
-        }
-        if (other.hasAggregatedAt()) {
-          setAggregatedAt(other.getAggregatedAt());
-        }
-        this.mergeUnknownFields(other.getUnknownFields());
-        return this;
-      }
-
-      public final boolean isInitialized() {
-        if (!hasDbName()) {
-          
-          return false;
-        }
-        if (!hasTableName()) {
-          
-          return false;
-        }
-        if (!hasBloomFilter()) {
-          
-          return false;
-        }
-        if (!hasAggregatedAt()) {
-          
-          return false;
-        }
-        if (!getBloomFilter().isInitialized()) {
-          
-          return false;
-        }
-        return true;
-      }
-
-      public Builder mergeFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws java.io.IOException {
-        org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parsedMessage = null;
-        try {
-          parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
-        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter) e.getUnfinishedMessage();
-          throw e;
-        } finally {
-          if (parsedMessage != null) {
-            mergeFrom(parsedMessage);
-          }
-        }
-        return this;
-      }
-      private int bitField0_;
-
-      // required bytes db_name = 1;
-      private com.google.protobuf.ByteString dbName_ = com.google.protobuf.ByteString.EMPTY;
-      /**
-       * <code>required bytes db_name = 1;</code>
-       */
-      public boolean hasDbName() {
-        return ((bitField0_ & 0x00000001) == 0x00000001);
-      }
-      /**
-       * <code>required bytes db_name = 1;</code>
-       */
-      public com.google.protobuf.ByteString getDbName() {
-        return dbName_;
-      }
-      /**
-       * <code>required bytes db_name = 1;</code>
-       */
-      public Builder setDbName(com.google.protobuf.ByteString value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  bitField0_ |= 0x00000001;
-        dbName_ = value;
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>required bytes db_name = 1;</code>
-       */
-      public Builder clearDbName() {
-        bitField0_ = (bitField0_ & ~0x00000001);
-        dbName_ = getDefaultInstance().getDbName();
-        onChanged();
-        return this;
-      }
-
-      // required bytes table_name = 2;
-      private com.google.protobuf.ByteString tableName_ = com.google.protobuf.ByteString.EMPTY;
-      /**
-       * <code>required bytes table_name = 2;</code>
-       */
-      public boolean hasTableName() {
-        return ((bitField0_ & 0x00000002) == 0x00000002);
-      }
-      /**
-       * <code>required bytes table_name = 2;</code>
-       */
-      public com.google.protobuf.ByteString getTableName() {
-        return tableName_;
-      }
-      /**
-       * <code>required bytes table_name = 2;</code>
-       */
-      public Builder setTableName(com.google.protobuf.ByteString value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  bitField0_ |= 0x00000002;
-        tableName_ = value;
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>required bytes table_name = 2;</code>
-       */
-      public Builder clearTableName() {
-        bitField0_ = (bitField0_ & ~0x00000002);
-        tableName_ = getDefaultInstance().getTableName();
-        onChanged();
-        return this;
-      }
-
-      // required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;
-      private org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter bloomFilter_ = org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
-      private com.google.protobuf.SingleFieldBuilder<
-          org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilterOrBuilder> bloomFilterBuilder_;
-      /**
-       * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-       */
-      public boolean hasBloomFilter() {
-        return ((bitField0_ & 0x00000004) == 0x00000004);
-      }
-      /**
-       * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-       */
-      public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter getBloomFilter() {
-        if (bloomFilterBuilder_ == null) {
-          return bloomFilter_;
-        } else {
-          return bloomFilterBuilder_.getMessage();
-        }
-      }
-      /**
-       * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-       */
-      public Builder setBloomFilter(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter value) {
-        if (bloomFilterBuilder_ == null) {
-          if (value == null) {
-            throw new NullPointerException();
-          }
-          bloomFilter_ = value;
-          onChanged();
-        } else {
-          bloomFilterBuilder_.setMessage(value);
-        }
-        bitField0_ |= 0x00000004;
-        return this;
-      }
-      /**
-       * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-       */
-      public Builder setBloomFilter(
-          org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder builderForValue) {
-        if (bloomFilterBuilder_ == null) {
-          bloomFilter_ = builderForValue.build();
-          onChanged();
-        } else {
-          bloomFilterBuilder_.setMessage(builderForValue.build());
-        }
-        bitField0_ |= 0x00000004;
-        return this;
-      }
-      /**
-       * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-       */
-      public Builder mergeBloomFilter(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter value) {
-        if (bloomFilterBuilder_ == null) {
-          if (((bitField0_ & 0x00000004) == 0x00000004) &&
-              bloomFilter_ != org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance()) {
-            bloomFilter_ =
-              org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.newBuilder(bloomFilter_).mergeFrom(value).buildPartial();
-          } else {
-            bloomFilter_ = value;
-          }
-          onChanged();
-        } else {
-          bloomFilterBuilder_.mergeFrom(value);
-        }
-        bitField0_ |= 0x00000004;
-        return this;
-      }
-      /**
-       * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
-       */
-      public Builder clearBloomFilter() {
-        if (bloomFilterBuilder_ == null) {
-          bloomFilter_ = org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
-          onChanged();
-        } else {
-          bloomFilterBuilder_.clear();
-        }
-        bitField0_ = (bitField0_ & ~0x00000004);
-        return this;
-      }
-      /**
-       * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter blo

<TRUNCATED>