You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/08/04 21:14:22 UTC
[01/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master c9c716246 -> 5e0615573
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreBitVector.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreBitVector.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreBitVector.java
deleted file mode 100644
index b1dc542..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreBitVector.java
+++ /dev/null
@@ -1,622 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class TestHBaseStoreBitVector {
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreBitVector.class.getName());
- static Map<String, String> emptyParameters = new HashMap<String, String>();
- // Table with NUM_PART_KEYS partitioning keys and NUM_PARTITIONS values per key
- static final int NUM_PART_KEYS = 1;
- static final int NUM_PARTITIONS = 5;
- static final String DB = "db";
- static final String TBL = "tbl";
- static final String COL = "col";
- static final String PART_KEY_PREFIX = "part";
- static final String PART_VAL_PREFIX = "val";
- static final String PART_KV_SEPARATOR = "=";
- static final List<String> PART_KEYS = new ArrayList<String>();
- static final List<String> PART_VALS = new ArrayList<String>();
- // Initialize mock partitions
- static {
- for (int i = 1; i <= NUM_PART_KEYS; i++) {
- PART_KEYS.add(PART_KEY_PREFIX + i);
- }
- for (int i = 1; i <= NUM_PARTITIONS; i++) {
- PART_VALS.add(PART_VAL_PREFIX + i);
- }
- }
- static final long DEFAULT_TIME = System.currentTimeMillis();
- static final String PART_KEY = "part";
- static final String LONG_COL = "longCol";
- static final String LONG_TYPE = "long";
- static final String INT_TYPE = "int";
- static final String INT_VAL = "1234";
- static final String DOUBLE_COL = "doubleCol";
- static final String DOUBLE_TYPE = "double";
- static final String DOUBLE_VAL = "3.1415";
- static final String STRING_COL = "stringCol";
- static final String STRING_TYPE = "string";
- static final String STRING_VAL = "stringval";
- static final String DECIMAL_COL = "decimalCol";
- static final String DECIMAL_TYPE = "decimal(5,3)";
- static final String DECIMAL_VAL = "12.123";
- static List<ColumnStatisticsObj> longColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
- static List<ColumnStatisticsObj> doubleColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
- static List<ColumnStatisticsObj> stringColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
- static List<ColumnStatisticsObj> decimalColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
-
- @Rule public ExpectedException thrown = ExpectedException.none();
- @Mock HTableInterface htable;
- SortedMap<String, Cell> rows = new TreeMap<>();
- HBaseStore store;
-
-
- @BeforeClass
- public static void beforeTest() {
- // All data intitializations
- populateMockStats();
- }
-
- private static void populateMockStats() {
- ColumnStatisticsObj statsObj;
- // Add NUM_PARTITIONS ColumnStatisticsObj of each type
- // For aggregate stats test, we'll treat each ColumnStatisticsObj as stats for 1 partition
- // For the rest, we'll just pick the 1st ColumnStatisticsObj from this list and use it
- for (int i = 0; i < NUM_PARTITIONS; i++) {
- statsObj = mockLongStats(i);
- longColStatsObjs.add(statsObj);
- statsObj = mockDoubleStats(i);
- doubleColStatsObjs.add(statsObj);
- statsObj = mockStringStats(i);
- stringColStatsObjs.add(statsObj);
- statsObj = mockDecimalStats(i);
- decimalColStatsObjs.add(statsObj);
- }
- }
-
- private static ColumnStatisticsObj mockLongStats(int i) {
- long high = 120938479124L + 100*i;
- long low = -12341243213412124L - 50*i;
- long nulls = 23 + i;
- long dVs = 213L + 10*i;
- String bitVectors = "{0, 1, 2, 3, 4, 5, 6, 7, 8}{1, 2, 3, 4, 5, 6, 7, 8}";
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(LONG_COL);
- colStatsObj.setColType(LONG_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData longData = new LongColumnStatsData();
- longData.setHighValue(high);
- longData.setLowValue(low);
- longData.setNumNulls(nulls);
- longData.setNumDVs(dVs);
- longData.setBitVectors(bitVectors);
- data.setLongStats(longData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- private static ColumnStatisticsObj mockDoubleStats(int i) {
- double high = 123423.23423 + 100*i;
- double low = 0.00001234233 - 50*i;
- long nulls = 92 + i;
- long dVs = 1234123421L + 10*i;
- String bitVectors = "{0, 1, 2, 3, 4, 5, 6, 7, 8}{0, 2, 3, 4, 5, 6, 7, 8}";
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(DOUBLE_COL);
- colStatsObj.setColType(DOUBLE_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- DoubleColumnStatsData doubleData = new DoubleColumnStatsData();
- doubleData.setHighValue(high);
- doubleData.setLowValue(low);
- doubleData.setNumNulls(nulls);
- doubleData.setNumDVs(dVs);
- doubleData.setBitVectors(bitVectors);
- data.setDoubleStats(doubleData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- private static ColumnStatisticsObj mockStringStats(int i) {
- long maxLen = 1234 + 10*i;
- double avgLen = 32.3 + i;
- long nulls = 987 + 10*i;
- long dVs = 906 + i;
- String bitVectors = "{0, 1, 2, 3, 4, 5, 6, 7, 8}{0, 1, 3, 4, 5, 6, 7, 8}";
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(STRING_COL);
- colStatsObj.setColType(STRING_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- StringColumnStatsData stringData = new StringColumnStatsData();
- stringData.setMaxColLen(maxLen);
- stringData.setAvgColLen(avgLen);
- stringData.setNumNulls(nulls);
- stringData.setNumDVs(dVs);
- stringData.setBitVectors(bitVectors);
- data.setStringStats(stringData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- private static ColumnStatisticsObj mockDecimalStats(int i) {
- Decimal high = new Decimal();
- high.setScale((short)3);
- String strHigh = String.valueOf(3876 + 100*i);
- high.setUnscaled(strHigh.getBytes());
- Decimal low = new Decimal();
- low.setScale((short)3);
- String strLow = String.valueOf(38 + i);
- low.setUnscaled(strLow.getBytes());
- long nulls = 13 + i;
- long dVs = 923947293L + 100*i;
- String bitVectors = "{0, 1, 2, 3, 4, 5, 6, 7, 8}{0, 1, 2, 4, 5, 6, 7, 8}";
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(DECIMAL_COL);
- colStatsObj.setColType(DECIMAL_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- DecimalColumnStatsData decimalData = new DecimalColumnStatsData();
- decimalData.setHighValue(high);
- decimalData.setLowValue(low);
- decimalData.setNumNulls(nulls);
- decimalData.setNumDVs(dVs);
- decimalData.setBitVectors(bitVectors);
- data.setDecimalStats(decimalData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- @AfterClass
- public static void afterTest() {
- }
-
-
- @Before
- public void init() throws IOException {
- MockitoAnnotations.initMocks(this);
- HiveConf conf = new HiveConf();
- conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
- store = MockUtils.init(conf, htable, rows);
- }
-
- @Test
- public void longTableStatistics() throws Exception {
- createMockTable(LONG_COL, LONG_TYPE);
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = longColStatsObjs.get(0);
- LongColumnStatsData longData = obj.getStatsData().getLongStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(LONG_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, dataFromDB.getSetField());
- // Compare LongColumnStatsData
- LongColumnStatsData longDataFromDB = dataFromDB.getLongStats();
- Assert.assertEquals(longData.getHighValue(), longDataFromDB.getHighValue());
- Assert.assertEquals(longData.getLowValue(), longDataFromDB.getLowValue());
- Assert.assertEquals(longData.getNumNulls(), longDataFromDB.getNumNulls());
- Assert.assertEquals(longData.getNumDVs(), longDataFromDB.getNumDVs());
- Assert.assertEquals(longData.getBitVectors(), longDataFromDB.getBitVectors());
- }
-
- @Test
- public void doubleTableStatistics() throws Exception {
- createMockTable(DOUBLE_COL, DOUBLE_TYPE);
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = doubleColStatsObjs.get(0);
- DoubleColumnStatsData doubleData = obj.getStatsData().getDoubleStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(DOUBLE_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, dataFromDB.getSetField());
- // Compare DoubleColumnStatsData
- DoubleColumnStatsData doubleDataFromDB = dataFromDB.getDoubleStats();
- Assert.assertEquals(doubleData.getHighValue(), doubleDataFromDB.getHighValue(), 0.01);
- Assert.assertEquals(doubleData.getLowValue(), doubleDataFromDB.getLowValue(), 0.01);
- Assert.assertEquals(doubleData.getNumNulls(), doubleDataFromDB.getNumNulls());
- Assert.assertEquals(doubleData.getNumDVs(), doubleDataFromDB.getNumDVs());
- Assert.assertEquals(doubleData.getBitVectors(), doubleDataFromDB.getBitVectors());
- }
-
- @Test
- public void stringTableStatistics() throws Exception {
- createMockTable(STRING_COL, STRING_TYPE);
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = stringColStatsObjs.get(0);
- StringColumnStatsData stringData = obj.getStatsData().getStringStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(STRING_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, dataFromDB.getSetField());
- // Compare StringColumnStatsData
- StringColumnStatsData stringDataFromDB = dataFromDB.getStringStats();
- Assert.assertEquals(stringData.getMaxColLen(), stringDataFromDB.getMaxColLen());
- Assert.assertEquals(stringData.getAvgColLen(), stringDataFromDB.getAvgColLen(), 0.01);
- Assert.assertEquals(stringData.getNumNulls(), stringDataFromDB.getNumNulls());
- Assert.assertEquals(stringData.getNumDVs(), stringDataFromDB.getNumDVs());
- Assert.assertEquals(stringData.getBitVectors(), stringDataFromDB.getBitVectors());
- }
-
- @Test
- public void decimalTableStatistics() throws Exception {
- createMockTable(DECIMAL_COL, DECIMAL_TYPE);
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = decimalColStatsObjs.get(0);
- DecimalColumnStatsData decimalData = obj.getStatsData().getDecimalStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(DECIMAL_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, dataFromDB.getSetField());
- // Compare DecimalColumnStatsData
- DecimalColumnStatsData decimalDataFromDB = dataFromDB.getDecimalStats();
- Assert.assertEquals(decimalData.getHighValue(), decimalDataFromDB.getHighValue());
- Assert.assertEquals(decimalData.getLowValue(), decimalDataFromDB.getLowValue());
- Assert.assertEquals(decimalData.getNumNulls(), decimalDataFromDB.getNumNulls());
- Assert.assertEquals(decimalData.getNumDVs(), decimalDataFromDB.getNumDVs());
- Assert.assertEquals(decimalData.getBitVectors(), decimalDataFromDB.getBitVectors());
- }
-
- @Test
- public void longPartitionStatistics() throws Exception {
- createMockTableAndPartition(INT_TYPE, INT_VAL);
- // Add partition stats for: LONG_COL and partition: {PART_KEY, INT_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, INT_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = longColStatsObjs.get(0);
- LongColumnStatsData longData = obj.getStatsData().getLongStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(INT_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, dataFromDB.getSetField());
- // Compare LongColumnStatsData
- LongColumnStatsData longDataFromDB = dataFromDB.getLongStats();
- Assert.assertEquals(longData.getHighValue(), longDataFromDB.getHighValue());
- Assert.assertEquals(longData.getLowValue(), longDataFromDB.getLowValue());
- Assert.assertEquals(longData.getNumNulls(), longDataFromDB.getNumNulls());
- Assert.assertEquals(longData.getNumDVs(), longDataFromDB.getNumDVs());
- Assert.assertEquals(longData.getBitVectors(), longDataFromDB.getBitVectors());
- }
-
- @Test
- public void doublePartitionStatistics() throws Exception {
- createMockTableAndPartition(DOUBLE_TYPE, DOUBLE_VAL);
- // Add partition stats for: DOUBLE_COL and partition: {PART_KEY, DOUBLE_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, DOUBLE_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = doubleColStatsObjs.get(0);
- DoubleColumnStatsData doubleData = obj.getStatsData().getDoubleStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(DOUBLE_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, dataFromDB.getSetField());
- // Compare DoubleColumnStatsData
- DoubleColumnStatsData doubleDataFromDB = dataFromDB.getDoubleStats();
- Assert.assertEquals(doubleData.getHighValue(), doubleDataFromDB.getHighValue(), 0.01);
- Assert.assertEquals(doubleData.getLowValue(), doubleDataFromDB.getLowValue(), 0.01);
- Assert.assertEquals(doubleData.getNumNulls(), doubleDataFromDB.getNumNulls());
- Assert.assertEquals(doubleData.getNumDVs(), doubleDataFromDB.getNumDVs());
- Assert.assertEquals(doubleData.getBitVectors(), doubleDataFromDB.getBitVectors());
- }
-
- @Test
- public void stringPartitionStatistics() throws Exception {
- createMockTableAndPartition(STRING_TYPE, STRING_VAL);
- // Add partition stats for: STRING_COL and partition: {PART_KEY, STRING_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, STRING_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = stringColStatsObjs.get(0);
- StringColumnStatsData stringData = obj.getStatsData().getStringStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(STRING_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, dataFromDB.getSetField());
- // Compare StringColumnStatsData
- StringColumnStatsData stringDataFromDB = dataFromDB.getStringStats();
- Assert.assertEquals(stringData.getMaxColLen(), stringDataFromDB.getMaxColLen());
- Assert.assertEquals(stringData.getAvgColLen(), stringDataFromDB.getAvgColLen(), 0.01);
- Assert.assertEquals(stringData.getNumNulls(), stringDataFromDB.getNumNulls());
- Assert.assertEquals(stringData.getNumDVs(), stringDataFromDB.getNumDVs());
- Assert.assertEquals(stringData.getBitVectors(), stringDataFromDB.getBitVectors());
- }
-
- @Test
- public void decimalPartitionStatistics() throws Exception {
- createMockTableAndPartition(DECIMAL_TYPE, DECIMAL_VAL);
- // Add partition stats for: DECIMAL_COL and partition: {PART_KEY, DECIMAL_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, DECIMAL_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = decimalColStatsObjs.get(0);
- DecimalColumnStatsData decimalData = obj.getStatsData().getDecimalStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(DECIMAL_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, dataFromDB.getSetField());
- // Compare DecimalColumnStatsData
- DecimalColumnStatsData decimalDataFromDB = dataFromDB.getDecimalStats();
- Assert.assertEquals(decimalData.getHighValue(), decimalDataFromDB.getHighValue());
- Assert.assertEquals(decimalData.getLowValue(), decimalDataFromDB.getLowValue());
- Assert.assertEquals(decimalData.getNumNulls(), decimalDataFromDB.getNumNulls());
- Assert.assertEquals(decimalData.getNumDVs(), decimalDataFromDB.getNumDVs());
- Assert.assertEquals(decimalData.getBitVectors(), decimalDataFromDB.getBitVectors());
- }
-
- private Table createMockTable(String name, String type) throws Exception {
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema(name, type, ""));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- Map<String, String> params = new HashMap<String, String>();
- params.put("key", "value");
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
- serde, new ArrayList<String>(), new ArrayList<Order>(), params);
- int currentTime = (int)(System.currentTimeMillis() / 1000);
- Table table = new Table(TBL, DB, "me", currentTime, currentTime, 0, sd, cols,
- emptyParameters, null, null, null);
- store.createTable(table);
- return table;
- }
-
- private Table createMockTableAndPartition(String partType, String partVal) throws Exception {
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", partType, ""));
- List<String> vals = new ArrayList<String>();
- vals.add(partVal);
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- Map<String, String> params = new HashMap<String, String>();
- params.put("key", "value");
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
- serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
- int currentTime = (int)(System.currentTimeMillis() / 1000);
- Table table = new Table(TBL, DB, "me", currentTime, currentTime, 0, sd, cols,
- emptyParameters, null, null, null);
- store.createTable(table);
- Partition part = new Partition(vals, DB, TBL, currentTime, currentTime, sd,
- emptyParameters);
- store.addPartition(part);
- return table;
- }
- /**
- * Returns a dummy table level ColumnStatisticsDesc with default values
- */
- private ColumnStatisticsDesc getMockTblColStatsDesc() {
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
- desc.setLastAnalyzed(DEFAULT_TIME);
- desc.setDbName(DB);
- desc.setTableName(TBL);
- desc.setIsTblLevel(true);
- return desc;
- }
-
- /**
- * Returns a dummy partition level ColumnStatisticsDesc
- */
- private ColumnStatisticsDesc getMockPartColStatsDesc(String partKey, String partVal) {
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
- desc.setLastAnalyzed(DEFAULT_TIME);
- desc.setDbName(DB);
- desc.setTableName(TBL);
- // part1=val1
- desc.setPartName(partKey + PART_KV_SEPARATOR + partVal);
- desc.setIsTblLevel(false);
- return desc;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
deleted file mode 100644
index 4ccb7dd..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreCached.java
+++ /dev/null
@@ -1,365 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class TestHBaseStoreCached {
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreCached.class.getName());
- static Map<String, String> emptyParameters = new HashMap<String, String>();
-
- @Rule public ExpectedException thrown = ExpectedException.none();
- @Mock HTableInterface htable;
- SortedMap<String, Cell> rows = new TreeMap<String, Cell>();
- HBaseStore store;
-
- @Before
- public void init() throws IOException {
- MockitoAnnotations.initMocks(this);
- HiveConf conf = new HiveConf();
- store = MockUtils.init(conf, htable, rows);
- }
-
- @Test
- public void createTable() throws Exception {
- String tableName = "mytable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- Table t = store.getTable("default", tableName);
- Assert.assertEquals(1, t.getSd().getColsSize());
- Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
- Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp", t.getSd().getLocation());
- Assert.assertEquals("input", t.getSd().getInputFormat());
- Assert.assertEquals("output", t.getSd().getOutputFormat());
- Assert.assertEquals("me", t.getOwner());
- Assert.assertEquals("default", t.getDbName());
- Assert.assertEquals(tableName, t.getTableName());
- }
-
- @Test
- public void alterTable() throws Exception {
- String tableName = "alttable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- startTime += 10;
- table.setLastAccessTime(startTime);
- store.alterTable("default", tableName, table);
-
- Table t = store.getTable("default", tableName);
- Assert.assertEquals(1, t.getSd().getColsSize());
- Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
- Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp", t.getSd().getLocation());
- Assert.assertEquals("input", t.getSd().getInputFormat());
- Assert.assertEquals("output", t.getSd().getOutputFormat());
- Assert.assertEquals("me", t.getOwner());
- Assert.assertEquals("default", t.getDbName());
- Assert.assertEquals(tableName, t.getTableName());
- Assert.assertEquals(startTime, t.getLastAccessTime());
- }
-
- @Test
- public void dropTable() throws Exception {
- String tableName = "dtable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- Table t = store.getTable("default", tableName);
- Assert.assertNotNull(t);
-
- store.dropTable("default", tableName);
- Assert.assertNull(store.getTable("default", tableName));
- }
-
- @Test
- public void createPartition() throws Exception {
- String dbName = "default";
- String tableName = "myparttable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> vals = Arrays.asList("fred");
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=fred");
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Partition p = store.getPartition(dbName, tableName, vals);
- Assert.assertEquals(1, p.getSd().getColsSize());
- Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
- Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
- Assert.assertEquals("input", p.getSd().getInputFormat());
- Assert.assertEquals("output", p.getSd().getOutputFormat());
- Assert.assertEquals(dbName, p.getDbName());
- Assert.assertEquals(tableName, p.getTableName());
- Assert.assertEquals(1, p.getValuesSize());
- Assert.assertEquals("fred", p.getValues().get(0));
- }
-
- @Test
- public void getPartitions() throws Exception {
- String dbName = "default";
- String tableName = "manyParts";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
- for (String val : partVals) {
- List<String> vals = new ArrayList<String>();
- vals.add(val);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=" + val);
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Partition p = store.getPartition(dbName, tableName, vals);
- Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
- }
-
- List<Partition> parts = store.getPartitions(dbName, tableName, -1);
- Assert.assertEquals(5, parts.size());
- String[] pv = new String[5];
- for (int i = 0; i < 5; i++) pv[i] = parts.get(i).getValues().get(0);
- Arrays.sort(pv);
- Assert.assertArrayEquals(pv, partVals.toArray(new String[5]));
- }
-
- @Test
- public void listGetDropPartitionNames() throws Exception {
- String dbName = "default";
- String tableName = "listParts";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- partCols.add(new FieldSchema("region", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- String[][] partVals = new String[][]{{"today", "north america"}, {"tomorrow", "europe"}};
- for (String[] pv : partVals) {
- List<String> vals = new ArrayList<String>();
- for (String v : pv) vals.add(v);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=" + pv[0] + "/region=" + pv[1]);
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
- }
-
- List<String> names = store.listPartitionNames(dbName, tableName, (short) -1);
- Assert.assertEquals(2, names.size());
- String[] resultNames = names.toArray(new String[names.size()]);
- Arrays.sort(resultNames);
- Assert.assertArrayEquals(resultNames, new String[]{"pc=today/region=north america",
- "pc=tomorrow/region=europe"});
-
- List<Partition> parts = store.getPartitionsByNames(dbName, tableName, names);
- Assert.assertArrayEquals(partVals[0], parts.get(0).getValues().toArray(new String[2]));
- Assert.assertArrayEquals(partVals[1], parts.get(1).getValues().toArray(new String[2]));
-
- store.dropPartitions(dbName, tableName, names);
- List<Partition> afterDropParts = store.getPartitions(dbName, tableName, -1);
- Assert.assertEquals(0, afterDropParts.size());
- }
-
-
- @Test
- public void dropPartition() throws Exception {
- String dbName = "default";
- String tableName = "myparttable2";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> vals = Arrays.asList("fred");
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=fred");
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Assert.assertNotNull(store.getPartition(dbName, tableName, vals));
- store.dropPartition(dbName, tableName, vals);
- thrown.expect(NoSuchObjectException.class);
- store.getPartition(dbName, tableName, vals);
- }
-
- // Due to the way our mock stuff works, we can only insert one column at a time, so we'll test
- // each stat type separately. We'll test them together in hte integration tests.
- @Test
- public void booleanTableStatistics() throws Exception {
- long now = System.currentTimeMillis();
- String dbname = "default";
- String tableName = "statstable";
- String boolcol = "boolcol";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema(boolcol, "boolean", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, dbname, "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- long trues = 37;
- long falses = 12;
- long booleanNulls = 2;
-
- ColumnStatistics stats = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
- desc.setLastAnalyzed(now);
- desc.setDbName(dbname);
- desc.setTableName(tableName);
- desc.setIsTblLevel(true);
- stats.setStatsDesc(desc);
-
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName(boolcol);
- obj.setColType("boolean");
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData boolData = new BooleanColumnStatsData();
- boolData.setNumTrues(trues);
- boolData.setNumFalses(falses);
- boolData.setNumNulls(booleanNulls);
- data.setBooleanStats(boolData);
- obj.setStatsData(data);
- stats.addToStatsObj(obj);
-
- store.updateTableColumnStatistics(stats);
-
- stats = store.getTableColumnStatistics(dbname, tableName, Arrays.asList(boolcol));
- Assert.assertEquals(now, stats.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(dbname, stats.getStatsDesc().getDbName());
- Assert.assertEquals(tableName, stats.getStatsDesc().getTableName());
- Assert.assertTrue(stats.getStatsDesc().isIsTblLevel());
-
- Assert.assertEquals(1, stats.getStatsObjSize());
- ColumnStatisticsData colData = obj.getStatsData();
- Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS, colData.getSetField());
- boolData = colData.getBooleanStats();
- Assert.assertEquals(trues, boolData.getNumTrues());
- Assert.assertEquals(falses, boolData.getNumFalses());
- Assert.assertEquals(booleanNulls, boolData.getNumNulls());
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java
deleted file mode 100644
index bea0b34..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestSharedStorageDescriptor.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- *
- */
-public class TestSharedStorageDescriptor {
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStore.class.getName());
-
-
- @Test
- public void changeOnSerde() {
- StorageDescriptor sd = new StorageDescriptor();
- SerDeInfo serde = new SerDeInfo();
- serde.setName("serde");
- sd.setSerdeInfo(serde);
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setShared(sd);
- ssd.getSerdeInfo().setName("different");
- Assert.assertFalse(sd.getSerdeInfo() == ssd.getSerdeInfo());
- Assert.assertEquals("serde", serde.getName());
- Assert.assertEquals("different", ssd.getSerdeInfo().getName());
- Assert.assertEquals("serde", sd.getSerdeInfo().getName());
- }
-
- @Test
- public void changeOnSkewed() {
- StorageDescriptor sd = new StorageDescriptor();
- SkewedInfo skew = new SkewedInfo();
- sd.setSkewedInfo(skew);
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setShared(sd);
- ssd.setSkewedInfo(new SkewedInfo());
- Assert.assertFalse(sd.getSkewedInfo() == ssd.getSkewedInfo());
- }
-
- @Test
- public void changeOnUnset() {
- StorageDescriptor sd = new StorageDescriptor();
- SkewedInfo skew = new SkewedInfo();
- sd.setSkewedInfo(skew);
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setShared(sd);
- ssd.unsetSkewedInfo();
- Assert.assertFalse(sd.getSkewedInfo() == ssd.getSkewedInfo());
- }
-
- @Test
- public void changeOrder() {
- StorageDescriptor sd = new StorageDescriptor();
- sd.addToSortCols(new Order("fred", 1));
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setShared(sd);
- ssd.getSortCols().get(0).setOrder(2);
- Assert.assertFalse(sd.getSortCols() == ssd.getSortCols());
- Assert.assertEquals(2, ssd.getSortCols().get(0).getOrder());
- Assert.assertEquals(1, sd.getSortCols().get(0).getOrder());
- }
-
- @Test
- public void unsetOrder() {
- StorageDescriptor sd = new StorageDescriptor();
- sd.addToSortCols(new Order("fred", 1));
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setShared(sd);
- ssd.unsetSortCols();
- Assert.assertFalse(sd.getSortCols() == ssd.getSortCols());
- Assert.assertEquals(0, ssd.getSortColsSize());
- Assert.assertEquals(1, sd.getSortColsSize());
- }
-
- @Test
- public void changeBucketList() {
- StorageDescriptor sd = new StorageDescriptor();
- sd.addToBucketCols(new String("fred"));
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setShared(sd);
- List<String> list = ssd.getBucketCols();
- list.add(new String("bob"));
- Assert.assertFalse(sd.getBucketCols() == ssd.getBucketCols());
- Assert.assertEquals(2, ssd.getBucketColsSize());
- Assert.assertEquals(1, sd.getBucketColsSize());
- }
-
- @Test
- public void addToColList() {
- StorageDescriptor sd = new StorageDescriptor();
- sd.addToCols(new FieldSchema("fred", "string", ""));
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setShared(sd);
- ssd.addToCols(new FieldSchema("joe", "int", ""));
- Assert.assertFalse(sd.getCols() == ssd.getCols());
- Assert.assertEquals(2, ssd.getColsSize());
- Assert.assertEquals(1, sd.getColsSize());
- }
-
- @Test
- public void colIterator() {
- StorageDescriptor sd = new StorageDescriptor();
- sd.addToCols(new FieldSchema("fred", "string", ""));
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setShared(sd);
- Iterator<FieldSchema> iter = ssd.getColsIterator();
- Assert.assertTrue(iter.hasNext());
- Assert.assertEquals("fred", iter.next().getName());
- Assert.assertFalse(sd.getCols() == ssd.getCols());
- }
-
- @Test
- public void setReadOnly() {
- StorageDescriptor sd = new StorageDescriptor();
- sd.addToCols(new FieldSchema("fred", "string", ""));
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setShared(sd);
- ssd.setReadOnly();
- List<FieldSchema> cols = ssd.getCols();
- Assert.assertEquals(1, cols.size());
- Assert.assertTrue(sd.getCols() == ssd.getCols());
- }
-
-}
[08/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
deleted file mode 100644
index d01f814..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ /dev/null
@@ -1,2969 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.CacheLoader;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.FileMetadataHandler;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.PartFilterExprUtil;
-import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.HiveObjectType;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.InvalidInputException;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionEventType;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TableMeta;
-import org.apache.hadoop.hive.metastore.api.Type;
-import org.apache.hadoop.hive.metastore.api.UnknownDBException;
-import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
-import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
-import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hive.common.util.HiveStringUtils;
-import org.apache.thrift.TException;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-/**
- * Implementation of RawStore that stores data in HBase
- */
-public class HBaseStore implements RawStore {
- static final private Logger LOG = LoggerFactory.getLogger(HBaseStore.class.getName());
-
- // Do not access this directly, call getHBase to make sure it is initialized.
- private HBaseReadWrite hbase = null;
- private Configuration conf;
- private int txnNestLevel = 0;
- private PartitionExpressionProxy expressionProxy = null;
- private Map<FileMetadataExprType, FileMetadataHandler> fmHandlers;
-
- public HBaseStore() {
- }
-
- @Override
- public void shutdown() {
- try {
- if (txnNestLevel != 0) rollbackTransaction();
- getHBase().close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public boolean openTransaction() {
- if (txnNestLevel++ <= 0) {
- LOG.debug("Opening HBase transaction");
- getHBase().begin();
- txnNestLevel = 1;
- }
- return true;
- }
-
- @Override
- public boolean commitTransaction() {
- if (--txnNestLevel == 0) {
- LOG.debug("Committing HBase transaction");
- getHBase().commit();
- }
- return true;
- }
-
- @Override
- public boolean isActiveTransaction() {
- return txnNestLevel != 0;
- }
-
- @Override
- public void rollbackTransaction() {
- txnNestLevel = 0;
- LOG.debug("Rolling back HBase transaction");
- getHBase().rollback();
- }
-
- @Override
- public void createDatabase(Database db) throws InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- Database dbCopy = db.deepCopy();
- dbCopy.setName(HiveStringUtils.normalizeIdentifier(dbCopy.getName()));
- // HiveMetaStore already checks for existence of the database, don't recheck
- getHBase().putDb(dbCopy);
- commit = true;
- } catch (IOException e) {
- LOG.error("Unable to create database ", e);
- throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
-
- }
-
- @Override
- public Database getDatabase(String name) throws NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- Database db = getHBase().getDb(HiveStringUtils.normalizeIdentifier(name));
- if (db == null) {
- throw new NoSuchObjectException("Unable to find db " + name);
- }
- commit = true;
- return db;
- } catch (IOException e) {
- LOG.error("Unable to get db", e);
- throw new NoSuchObjectException("Error reading db " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- getHBase().deleteDb(HiveStringUtils.normalizeIdentifier(dbname));
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to delete db" + e);
- throw new MetaException("Unable to drop database " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean alterDatabase(String dbname, Database db) throws NoSuchObjectException,
- MetaException {
- // ObjectStore fetches the old db before updating it, but I can't see the possible value of
- // that since the caller will have needed to call getDatabase to have the db object.
- boolean commit = false;
- openTransaction();
- try {
- Database dbCopy = db.deepCopy();
- dbCopy.setName(HiveStringUtils.normalizeIdentifier(dbCopy.getName()));
- getHBase().putDb(dbCopy);
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to alter database ", e);
- throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<String> getDatabases(String pattern) throws MetaException {
- boolean commit = false;
- openTransaction();
- try {
- List<Database> dbs = getHBase().scanDatabases(
- pattern==null?null:HiveStringUtils.normalizeIdentifier(likeToRegex(pattern)));
- List<String> dbNames = new ArrayList<String>(dbs.size());
- for (Database db : dbs) dbNames.add(db.getName());
- commit = true;
- return dbNames;
- } catch (IOException e) {
- LOG.error("Unable to get databases ", e);
- throw new MetaException("Unable to get databases, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<String> getAllDatabases() throws MetaException {
- return getDatabases(null);
- }
-
- @Override
- public int getDatabaseCount() throws MetaException {
- try {
- return getHBase().getDatabaseCount();
- } catch (IOException e) {
- LOG.error("Unable to get database count", e);
- throw new MetaException("Error scanning databases");
- }
- }
-
- @Override
- public boolean createType(Type type) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Type getType(String typeName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean dropType(String typeName) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void createTable(Table tbl) throws InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- // HiveMetaStore above us checks if the table already exists, so we can blindly store it here.
- try {
- Table tblCopy = tbl.deepCopy();
- tblCopy.setDbName(HiveStringUtils.normalizeIdentifier(tblCopy.getDbName()));
- tblCopy.setTableName(HiveStringUtils.normalizeIdentifier(tblCopy.getTableName()));
- normalizeColumnNames(tblCopy);
- getHBase().putTable(tblCopy);
- commit = true;
- } catch (IOException e) {
- LOG.error("Unable to create table ", e);
- throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- private void normalizeColumnNames(Table tbl) {
- if (tbl.getSd().getCols() != null) {
- tbl.getSd().setCols(normalizeFieldSchemaList(tbl.getSd().getCols()));
- }
- if (tbl.getPartitionKeys() != null) {
- tbl.setPartitionKeys(normalizeFieldSchemaList(tbl.getPartitionKeys()));
- }
- }
-
- private List<FieldSchema> normalizeFieldSchemaList(List<FieldSchema> fieldschemas) {
- List<FieldSchema> ret = new ArrayList<>();
- for (FieldSchema fieldSchema : fieldschemas) {
- ret.add(new FieldSchema(fieldSchema.getName().toLowerCase(), fieldSchema.getType(),
- fieldSchema.getComment()));
- }
- return ret;
- }
-
- @Override
- public boolean dropTable(String dbName, String tableName) throws MetaException,
- NoSuchObjectException, InvalidObjectException, InvalidInputException {
- boolean commit = false;
- openTransaction();
- try {
- getHBase().deleteTable(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName));
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to delete db" + e);
- throw new MetaException("Unable to drop table " + tableNameForErrorMsg(dbName, tableName));
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public Table getTable(String dbName, String tableName) throws MetaException {
- boolean commit = false;
- openTransaction();
- try {
- Table table = getHBase().getTable(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName));
- if (table == null) {
- LOG.debug("Unable to find table " + tableNameForErrorMsg(dbName, tableName));
- }
- commit = true;
- return table;
- } catch (IOException e) {
- LOG.error("Unable to get table", e);
- throw new MetaException("Error reading table " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean addPartition(Partition part) throws InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- Partition partCopy = part.deepCopy();
- partCopy.setDbName(HiveStringUtils.normalizeIdentifier(part.getDbName()));
- partCopy.setTableName(HiveStringUtils.normalizeIdentifier(part.getTableName()));
- getHBase().putPartition(partCopy);
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to add partition", e);
- throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean addPartitions(String dbName, String tblName, List<Partition> parts)
- throws InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- List<Partition> partsCopy = new ArrayList<Partition>();
- for (int i=0;i<parts.size();i++) {
- Partition partCopy = parts.get(i).deepCopy();
- partCopy.setDbName(HiveStringUtils.normalizeIdentifier(partCopy.getDbName()));
- partCopy.setTableName(HiveStringUtils.normalizeIdentifier(partCopy.getTableName()));
- partsCopy.add(i, partCopy);
- }
- getHBase().putPartitions(partsCopy);
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to add partitions", e);
- throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean addPartitions(String dbName, String tblName, PartitionSpecProxy partitionSpec,
- boolean ifNotExists) throws InvalidObjectException, MetaException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Partition getPartition(String dbName, String tableName, List<String> part_vals) throws
- MetaException, NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- Partition part = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName), part_vals);
- if (part == null) {
- throw new NoSuchObjectException("Unable to find partition " +
- partNameForErrorMsg(dbName, tableName, part_vals));
- }
- commit = true;
- return part;
- } catch (IOException e) {
- LOG.error("Unable to get partition", e);
- throw new MetaException("Error reading partition " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean doesPartitionExist(String dbName, String tableName, List<String> part_vals) throws
- MetaException, NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- boolean exists = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName), part_vals) != null;
- commit = true;
- return exists;
- } catch (IOException e) {
- LOG.error("Unable to get partition", e);
- throw new MetaException("Error reading partition " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean dropPartition(String dbName, String tableName, List<String> part_vals) throws
- MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
- boolean commit = false;
- openTransaction();
- try {
- dbName = HiveStringUtils.normalizeIdentifier(dbName);
- tableName = HiveStringUtils.normalizeIdentifier(tableName);
- getHBase().deletePartition(dbName, tableName, HBaseUtils.getPartitionKeyTypes(
- getTable(dbName, tableName).getPartitionKeys()), part_vals);
- // Drop any cached stats that reference this partitions
- getHBase().getStatsCache().invalidate(dbName, tableName,
- buildExternalPartName(dbName, tableName, part_vals));
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to delete db" + e);
- throw new MetaException("Unable to drop partition " + partNameForErrorMsg(dbName, tableName,
- part_vals));
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<Partition> getPartitions(String dbName, String tableName, int max) throws
- MetaException, NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- List<Partition> parts = getHBase().scanPartitionsInTable(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName), max);
- commit = true;
- return parts;
- } catch (IOException e) {
- LOG.error("Unable to get partitions", e);
- throw new MetaException("Error scanning partitions");
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public void alterTable(String dbName, String tableName, Table newTable) throws InvalidObjectException,
- MetaException {
- boolean commit = false;
- openTransaction();
- try {
- Table newTableCopy = newTable.deepCopy();
- newTableCopy.setDbName(HiveStringUtils.normalizeIdentifier(newTableCopy.getDbName()));
- List<String> oldPartTypes = getTable(dbName, tableName).getPartitionKeys()==null?
- null:HBaseUtils.getPartitionKeyTypes(getTable(dbName, tableName).getPartitionKeys());
- newTableCopy.setTableName(HiveStringUtils.normalizeIdentifier(newTableCopy.getTableName()));
- getHBase().replaceTable(getHBase().getTable(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName)), newTableCopy);
- if (newTable.getPartitionKeys() != null && newTable.getPartitionKeys().size() > 0
- && !tableName.equals(newTable.getTableName())) {
- // They renamed the table, so we need to change each partition as well, since it changes
- // the key.
- try {
- List<Partition> oldParts = getPartitions(dbName, tableName, -1);
- List<Partition> newParts = new ArrayList<>(oldParts.size());
- for (Partition oldPart : oldParts) {
- Partition newPart = oldPart.deepCopy();
- newPart.setTableName(newTable.getTableName());
- newParts.add(newPart);
- }
- getHBase().replacePartitions(oldParts, newParts, oldPartTypes);
- } catch (NoSuchObjectException e) {
- LOG.debug("No partitions found for old table so not worrying about it");
- }
-
- }
- commit = true;
- } catch (IOException e) {
- LOG.error("Unable to alter table " + tableNameForErrorMsg(dbName, tableName), e);
- throw new MetaException("Unable to alter table " + tableNameForErrorMsg(dbName, tableName));
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<String> getTables(String dbName, String pattern) throws MetaException {
- boolean commit = false;
- openTransaction();
- try {
- List<String> tableNames = getTableNamesInTx(dbName, pattern);
- commit = true;
- return tableNames;
- } catch (IOException e) {
- LOG.error("Unable to get tables ", e);
- throw new MetaException("Unable to get tables, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<String> getTables(String dbName, String pattern, TableType tableType) throws MetaException {
- throw new UnsupportedOperationException();
- }
-
- private List<String> getTableNamesInTx(String dbName, String pattern) throws IOException {
- List<Table> tables = getHBase().scanTables(HiveStringUtils.normalizeIdentifier(dbName),
- pattern==null?null:HiveStringUtils.normalizeIdentifier(likeToRegex(pattern)));
- List<String> tableNames = new ArrayList<String>(tables.size());
- for (Table table : tables) tableNames.add(table.getTableName());
- return tableNames;
- }
-
- @Override
- public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes)
- throws MetaException {
- boolean commit = false;
- openTransaction();
- try {
- List<TableMeta> metas = new ArrayList<>();
- for (String dbName : getDatabases(dbNames)) {
- for (Table table : getTableObjectsByName(dbName, getTableNamesInTx(dbName, tableNames))) {
- if (tableTypes == null || tableTypes.contains(table.getTableType())) {
- TableMeta metaData = new TableMeta(
- table.getDbName(), table.getTableName(), table.getTableType());
- metaData.setComments(table.getParameters().get("comment"));
- metas.add(metaData);
- }
- }
- }
- commit = true;
- return metas;
- } catch (Exception e) {
- LOG.error("Unable to get tables ", e);
- throw new MetaException("Unable to get tables, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<Table> getTableObjectsByName(String dbname, List<String> tableNames) throws
- MetaException, UnknownDBException {
- boolean commit = false;
- openTransaction();
- try {
- List<String> normalizedTableNames = new ArrayList<String>(tableNames.size());
- for (String tableName : tableNames) {
- normalizedTableNames.add(HiveStringUtils.normalizeIdentifier(tableName));
- }
- List<Table> tables = getHBase().getTables(HiveStringUtils.normalizeIdentifier(dbname),
- normalizedTableNames);
- commit = true;
- return tables;
- } catch (IOException e) {
- LOG.error("Unable to get tables ", e);
- throw new MetaException("Unable to get tables, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<String> getAllTables(String dbName) throws MetaException {
- return getTables(dbName, null);
- }
-
- @Override
- public int getTableCount() throws MetaException {
- try {
- return getHBase().getTableCount();
- } catch (IOException e) {
- LOG.error("Unable to get table count", e);
- throw new MetaException("Error scanning tables");
- }
- }
-
- @Override
- public List<String> listTableNamesByFilter(String dbName, String filter, short max_tables) throws
- MetaException, UnknownDBException {
- // TODO needs to wait until we support pushing filters into HBase.
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<String> listPartitionNames(String db_name, String tbl_name, short max_parts) throws
- MetaException {
- boolean commit = false;
- openTransaction();
- try {
- List<Partition> parts = getHBase().scanPartitionsInTable(HiveStringUtils.normalizeIdentifier(db_name),
- HiveStringUtils.normalizeIdentifier(tbl_name), max_parts);
- if (parts == null) return null;
- List<String> names = new ArrayList<String>(parts.size());
- Table table = getHBase().getTable(HiveStringUtils.normalizeIdentifier(db_name),
- HiveStringUtils.normalizeIdentifier(tbl_name));
- for (Partition p : parts) {
- names.add(buildExternalPartName(table, p));
- }
- commit = true;
- return names;
- } catch (IOException e) {
- LOG.error("Unable to get partitions", e);
- throw new MetaException("Error scanning partitions");
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<String> listPartitionNamesByFilter(String db_name, String tbl_name, String filter,
- short max_parts) throws MetaException {
- // TODO needs to wait until we support pushing filters into HBase.
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void alterPartition(String db_name, String tbl_name, List<String> part_vals,
- Partition new_part) throws InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- Partition new_partCopy = new_part.deepCopy();
- new_partCopy.setDbName(HiveStringUtils.normalizeIdentifier(new_partCopy.getDbName()));
- new_partCopy.setTableName(HiveStringUtils.normalizeIdentifier(new_partCopy.getTableName()));
- Partition oldPart = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(db_name),
- HiveStringUtils.normalizeIdentifier(tbl_name), part_vals);
- getHBase().replacePartition(oldPart, new_partCopy, HBaseUtils.getPartitionKeyTypes(
- getTable(db_name, tbl_name).getPartitionKeys()));
- // Drop any cached stats that reference this partitions
- getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(db_name),
- HiveStringUtils.normalizeIdentifier(tbl_name),
- buildExternalPartName(db_name, tbl_name, part_vals));
- commit = true;
- } catch (IOException e) {
- LOG.error("Unable to add partition", e);
- throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public void alterPartitions(String db_name, String tbl_name, List<List<String>> part_vals_list,
- List<Partition> new_parts) throws InvalidObjectException,
- MetaException {
- boolean commit = false;
- openTransaction();
- try {
- List<Partition> new_partsCopy = new ArrayList<Partition>();
- for (int i=0;i<new_parts.size();i++) {
- Partition newPartCopy = new_parts.get(i).deepCopy();
- newPartCopy.setDbName(HiveStringUtils.normalizeIdentifier(newPartCopy.getDbName()));
- newPartCopy.setTableName(HiveStringUtils.normalizeIdentifier(newPartCopy.getTableName()));
- new_partsCopy.add(i, newPartCopy);
- }
- List<Partition> oldParts = getHBase().getPartitions(HiveStringUtils.normalizeIdentifier(db_name),
- HiveStringUtils.normalizeIdentifier(tbl_name),
- HBaseUtils.getPartitionKeyTypes(getTable(HiveStringUtils.normalizeIdentifier(db_name),
- HiveStringUtils.normalizeIdentifier(tbl_name)).getPartitionKeys()), part_vals_list);
- getHBase().replacePartitions(oldParts, new_partsCopy, HBaseUtils.getPartitionKeyTypes(
- getTable(db_name, tbl_name).getPartitionKeys()));
- for (List<String> part_vals : part_vals_list) {
- getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(db_name),
- HiveStringUtils.normalizeIdentifier(tbl_name),
- buildExternalPartName(db_name, tbl_name, part_vals));
- }
- commit = true;
- } catch (IOException e) {
- LOG.error("Unable to add partition", e);
- throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean addIndex(Index index) throws InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- index.setDbName(HiveStringUtils.normalizeIdentifier(index.getDbName()));
- index.setOrigTableName(HiveStringUtils.normalizeIdentifier(index.getOrigTableName()));
- index.setIndexName(HiveStringUtils.normalizeIdentifier(index.getIndexName()));
- index.setIndexTableName(HiveStringUtils.normalizeIdentifier(index.getIndexTableName()));
- getHBase().putIndex(index);
- commit = true;
- } catch (IOException e) {
- LOG.error("Unable to create index ", e);
- throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- return commit;
- }
-
- @Override
- public Index getIndex(String dbName, String origTableName, String indexName) throws
- MetaException {
- boolean commit = false;
- openTransaction();
- try {
- Index index = getHBase().getIndex(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(origTableName),
- HiveStringUtils.normalizeIdentifier(indexName));
- if (index == null) {
- LOG.debug("Unable to find index " + indexNameForErrorMsg(dbName, origTableName, indexName));
- }
- commit = true;
- return index;
- } catch (IOException e) {
- LOG.error("Unable to get index", e);
- throw new MetaException("Error reading index " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean dropIndex(String dbName, String origTableName, String indexName) throws
- MetaException {
- boolean commit = false;
- openTransaction();
- try {
- getHBase().deleteIndex(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(origTableName),
- HiveStringUtils.normalizeIdentifier(indexName));
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to delete index" + e);
- throw new MetaException("Unable to drop index "
- + indexNameForErrorMsg(dbName, origTableName, indexName));
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<Index> getIndexes(String dbName, String origTableName, int max) throws MetaException {
- boolean commit = false;
- openTransaction();
- try {
- List<Index> indexes = getHBase().scanIndexes(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(origTableName), max);
- commit = true;
- return indexes;
- } catch (IOException e) {
- LOG.error("Unable to get indexes", e);
- throw new MetaException("Error scanning indexxes");
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<String> listIndexNames(String dbName, String origTableName, short max) throws
- MetaException {
- boolean commit = false;
- openTransaction();
- try {
- List<Index> indexes = getHBase().scanIndexes(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(origTableName), max);
- if (indexes == null) return null;
- List<String> names = new ArrayList<String>(indexes.size());
- for (Index index : indexes) {
- names.add(index.getIndexName());
- }
- commit = true;
- return names;
- } catch (IOException e) {
- LOG.error("Unable to get indexes", e);
- throw new MetaException("Error scanning indexes");
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public void alterIndex(String dbname, String baseTblName, String name, Index newIndex) throws
- InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- Index newIndexCopy = newIndex.deepCopy();
- newIndexCopy.setDbName(HiveStringUtils.normalizeIdentifier(newIndexCopy.getDbName()));
- newIndexCopy.setOrigTableName(
- HiveStringUtils.normalizeIdentifier(newIndexCopy.getOrigTableName()));
- newIndexCopy.setIndexName(HiveStringUtils.normalizeIdentifier(newIndexCopy.getIndexName()));
- getHBase().replaceIndex(getHBase().getIndex(HiveStringUtils.normalizeIdentifier(dbname),
- HiveStringUtils.normalizeIdentifier(baseTblName),
- HiveStringUtils.normalizeIdentifier(name)), newIndexCopy);
- commit = true;
- } catch (IOException e) {
- LOG.error("Unable to alter index " + indexNameForErrorMsg(dbname, baseTblName, name), e);
- throw new MetaException("Unable to alter index "
- + indexNameForErrorMsg(dbname, baseTblName, name));
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<Partition> getPartitionsByFilter(String dbName, String tblName, String filter,
- short maxParts) throws MetaException,
- NoSuchObjectException {
- final ExpressionTree exprTree = (filter != null && !filter.isEmpty()) ? PartFilterExprUtil
- .getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE;
- List<Partition> result = new ArrayList<Partition>();
- boolean commit = false;
- openTransaction();
- try {
- getPartitionsByExprInternal(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tblName), exprTree, maxParts, result);
- return result;
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr,
- String defaultPartitionName, short maxParts,
- List<Partition> result) throws TException {
- final ExpressionTree exprTree = PartFilterExprUtil.makeExpressionTree(expressionProxy, expr);
- dbName = HiveStringUtils.normalizeIdentifier(dbName);
- tblName = HiveStringUtils.normalizeIdentifier(tblName);
- Table table = getTable(dbName, tblName);
- boolean commit = false;
- openTransaction();
- try {
- if (exprTree == null) {
- List<String> partNames = new LinkedList<String>();
- boolean hasUnknownPartitions = getPartitionNamesPrunedByExprNoTxn(
- table, expr, defaultPartitionName, maxParts, partNames);
- result.addAll(getPartitionsByNames(dbName, tblName, partNames));
- return hasUnknownPartitions;
- } else {
- return getPartitionsByExprInternal(dbName, tblName, exprTree, maxParts, result);
- }
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public int getNumPartitionsByFilter(String dbName, String tblName, String filter)
- throws MetaException, NoSuchObjectException {
- final ExpressionTree exprTree = (filter != null && !filter.isEmpty()) ? PartFilterExprUtil
- .getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE;
- List<Partition> result = new ArrayList<Partition>();
- boolean commit = false;
- openTransaction();
- try {
- return getPartitionsByFilter(dbName, tblName, filter, Short.MAX_VALUE).size();
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public int getNumPartitionsByExpr(String dbName, String tblName, byte[] expr)
- throws MetaException, NoSuchObjectException {
- final ExpressionTree exprTree = PartFilterExprUtil.makeExpressionTree(expressionProxy, expr);
- List<Partition> result = new ArrayList<Partition>();
- boolean commit = false;
- openTransaction();
- try {
- getPartitionsByExprInternal(dbName, tblName, exprTree, Short.MAX_VALUE, result);
- return result.size();
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- /**
- * Gets the partition names from a table, pruned using an expression.
- * @param table Table.
- * @param expr Expression.
- * @param defaultPartName Default partition name from job config, if any.
- * @param maxParts Maximum number of partition names to return.
- * @param result The resulting names.
- * @return Whether the result contains any unknown partitions.
- * @throws NoSuchObjectException
- */
- private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr,
- String defaultPartName, short maxParts, List<String> result) throws MetaException, NoSuchObjectException {
- List<Partition> parts = getPartitions(
- table.getDbName(), table.getTableName(), maxParts);
- for (Partition part : parts) {
- result.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues()));
- }
- List<String> columnNames = new ArrayList<String>();
- List<PrimitiveTypeInfo> typeInfos = new ArrayList<PrimitiveTypeInfo>();
- for (FieldSchema fs : table.getPartitionKeys()) {
- columnNames.add(fs.getName());
- typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
- }
- if (defaultPartName == null || defaultPartName.isEmpty()) {
- defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME);
- }
- return expressionProxy.filterPartitionsByExpr(
- columnNames, typeInfos, expr, defaultPartName, result);
- }
-
- private boolean getPartitionsByExprInternal(String dbName, String tblName,
- ExpressionTree exprTree, short maxParts, List<Partition> result) throws MetaException,
- NoSuchObjectException {
-
- dbName = HiveStringUtils.normalizeIdentifier(dbName);
- tblName = HiveStringUtils.normalizeIdentifier(tblName);
- Table table = getTable(dbName, tblName);
- if (table == null) {
- throw new NoSuchObjectException("Unable to find table " + dbName + "." + tblName);
- }
- // general hbase filter plan from expression tree
- PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, table.getPartitionKeys());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Hbase Filter Plan generated : " + planRes.plan);
- }
-
- // results from scans need to be merged as there can be overlapping results between
- // the scans. Use a map of list of partition values to partition for this.
- Map<List<String>, Partition> mergedParts = new HashMap<List<String>, Partition>();
- for (ScanPlan splan : planRes.plan.getPlans()) {
- try {
- List<Partition> parts = getHBase().scanPartitions(dbName, tblName,
- splan.getStartRowSuffix(dbName, tblName, table.getPartitionKeys()),
- splan.getEndRowSuffix(dbName, tblName, table.getPartitionKeys()),
- splan.getFilter(table.getPartitionKeys()), -1);
- boolean reachedMax = false;
- for (Partition part : parts) {
- mergedParts.put(part.getValues(), part);
- if (mergedParts.size() == maxParts) {
- reachedMax = true;
- break;
- }
- }
- if (reachedMax) {
- break;
- }
- } catch (IOException e) {
- LOG.error("Unable to get partitions", e);
- throw new MetaException("Error scanning partitions" + tableNameForErrorMsg(dbName, tblName)
- + ": " + e);
- }
- }
- for (Entry<List<String>, Partition> mp : mergedParts.entrySet()) {
- result.add(mp.getValue());
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Matched partitions " + result);
- }
-
- // return true if there might be some additional partitions that don't match filter conditions
- // being returned
- return !planRes.hasUnsupportedCondition;
- }
-
- @Override
- public List<Partition> getPartitionsByNames(String dbName, String tblName,
- List<String> partNames) throws MetaException,
- NoSuchObjectException {
- List<Partition> parts = new ArrayList<Partition>();
- for (String partName : partNames) {
- parts.add(getPartition(dbName, tblName, partNameToVals(partName)));
- }
- return parts;
- }
-
- @Override
- public Table markPartitionForEvent(String dbName, String tblName, Map<String, String> partVals,
- PartitionEventType evtType) throws MetaException,
- UnknownTableException, InvalidPartitionException, UnknownPartitionException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean isPartitionMarkedForEvent(String dbName, String tblName,
- Map<String, String> partName,
- PartitionEventType evtType) throws MetaException,
- UnknownTableException, InvalidPartitionException, UnknownPartitionException {
- throw new UnsupportedOperationException();
- }
-
- /*
- * The design for roles. Roles are a pain because of their hierarchical nature. When a user
- * comes in and we need to be able to determine all roles he is a part of, we do not want to
- * have to walk the hierarchy in the database. This means we need to flatten the role map for
- * each user. But we also have to track how the roles are connected for each user, in case one
- * role is revoked from another (e.g. if role1 is included in role2 but then revoked
- * from it and user1 was granted both role2 and role1 we cannot remove user1 from role1
- * because he was granted that separately).
- *
- * We want to optimize for the read case and put the cost on grant and revoke of roles, since
- * we assume that is the less common case. So we lay out the roles data as follows:
- *
- * There is a ROLES table that records each role, plus what other principals have been granted
- * into it, along with the info on grantor, etc.
- *
- * There is a USER_TO_ROLES table that contains the mapping of each user to every role he is a
- * part of.
- *
- * This makes determining what roles a user participates in very quick, as USER_TO_ROLE is a
- * simple list for each user. It makes granting users into roles expensive, and granting roles
- * into roles very expensive. Each time a user is granted into a role, we need to walk the
- * hierarchy in the role table (which means moving through that table multiple times) to
- * determine every role the user participates in. Each a role is granted into another role
- * this hierarchical walk must be done for every principal in the role being granted into. To
- * mitigate this pain somewhat whenever doing these mappings we cache the entire ROLES table in
- * memory since we assume it is not large.
- *
- * On a related note, whenever a role is dropped we must walk not only all these role tables
- * above (equivalent to a role being revoked from another role, since we have to rebuilding
- * mappings for any users in roles that contained that role and any users directly in that
- * role), but we also have to remove all the privileges associated with that role directly.
- * That means a walk of the DBS table and of the TBLS table.
- */
-
- @Override
- public int getPartitionCount() throws MetaException {
- try {
- return getHBase().getPartitionCount();
- } catch (IOException e) {
- LOG.error("Unable to get partition count", e);
- throw new MetaException("Error scanning partitions");
- }
- }
-
- @Override
- public boolean addRole(String roleName, String ownerName) throws InvalidObjectException,
- MetaException, NoSuchObjectException {
- int now = (int)(System.currentTimeMillis()/1000);
- Role role = new Role(roleName, now, ownerName);
- boolean commit = false;
- openTransaction();
- try {
- if (getHBase().getRole(roleName) != null) {
- throw new InvalidObjectException("Role " + roleName + " already exists");
- }
- getHBase().putRole(role);
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to create role ", e);
- throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean removeRole(String roleName) throws MetaException, NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- Set<String> usersInRole = getHBase().findAllUsersInRole(roleName);
- getHBase().deleteRole(roleName);
- getHBase().removeRoleGrants(roleName);
- for (String user : usersInRole) {
- getHBase().buildRoleMapForUser(user);
- }
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to delete role" + e);
- throw new MetaException("Unable to drop role " + roleName);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean grantRole(Role role, String userName, PrincipalType principalType, String grantor,
- PrincipalType grantorType, boolean grantOption)
- throws MetaException, NoSuchObjectException, InvalidObjectException {
- boolean commit = false;
- openTransaction();
- try {
- Set<String> usersToRemap = findUsersToRemapRolesFor(role, userName, principalType);
- HbaseMetastoreProto.RoleGrantInfo.Builder builder =
- HbaseMetastoreProto.RoleGrantInfo.newBuilder();
- if (userName != null) builder.setPrincipalName(userName);
- if (principalType != null) {
- builder.setPrincipalType(HBaseUtils.convertPrincipalTypes(principalType));
- }
- builder.setAddTime((int)(System.currentTimeMillis() / 1000));
- if (grantor != null) builder.setGrantor(grantor);
- if (grantorType != null) {
- builder.setGrantorType(HBaseUtils.convertPrincipalTypes(grantorType));
- }
- builder.setGrantOption(grantOption);
-
- getHBase().addPrincipalToRole(role.getRoleName(), builder.build());
- for (String user : usersToRemap) {
- getHBase().buildRoleMapForUser(user);
- }
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to grant role", e);
- throw new MetaException("Unable to grant role " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean revokeRole(Role role, String userName, PrincipalType principalType,
- boolean grantOption) throws MetaException, NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- // This can have a couple of different meanings. If grantOption is true, then this is only
- // revoking the grant option, the role itself doesn't need to be removed. If it is false
- // then we need to remove the userName from the role altogether.
- try {
- if (grantOption) {
- // If this is a grant only change, we don't need to rebuild the user mappings.
- getHBase().dropPrincipalFromRole(role.getRoleName(), userName, principalType, grantOption);
- } else {
- Set<String> usersToRemap = findUsersToRemapRolesFor(role, userName, principalType);
- getHBase().dropPrincipalFromRole(role.getRoleName(), userName, principalType, grantOption);
- for (String user : usersToRemap) {
- getHBase().buildRoleMapForUser(user);
- }
- }
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to revoke role " + role.getRoleName() + " from " + userName, e);
- throw new MetaException("Unable to revoke role " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public PrincipalPrivilegeSet getUserPrivilegeSet(String userName, List<String> groupNames)
- throws InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- PrincipalPrivilegeSet pps = new PrincipalPrivilegeSet();
- PrincipalPrivilegeSet global = getHBase().getGlobalPrivs();
- if (global == null) return null;
- List<PrivilegeGrantInfo> pgi;
- if (global.getUserPrivileges() != null) {
- pgi = global.getUserPrivileges().get(userName);
- if (pgi != null) {
- pps.putToUserPrivileges(userName, pgi);
- }
- }
-
- if (global.getRolePrivileges() != null) {
- List<String> roles = getHBase().getUserRoles(userName);
- if (roles != null) {
- for (String role : roles) {
- pgi = global.getRolePrivileges().get(role);
- if (pgi != null) {
- pps.putToRolePrivileges(role, pgi);
- }
- }
- }
- }
- commit = true;
- return pps;
- } catch (IOException e) {
- LOG.error("Unable to get db privileges for user", e);
- throw new MetaException("Unable to get db privileges for user, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public PrincipalPrivilegeSet getDBPrivilegeSet(String dbName, String userName,
- List<String> groupNames)
- throws InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- PrincipalPrivilegeSet pps = new PrincipalPrivilegeSet();
- Database db = getHBase().getDb(dbName);
- if (db.getPrivileges() != null) {
- List<PrivilegeGrantInfo> pgi;
- // Find the user privileges for this db
- if (db.getPrivileges().getUserPrivileges() != null) {
- pgi = db.getPrivileges().getUserPrivileges().get(userName);
- if (pgi != null) {
- pps.putToUserPrivileges(userName, pgi);
- }
- }
-
- if (db.getPrivileges().getRolePrivileges() != null) {
- List<String> roles = getHBase().getUserRoles(userName);
- if (roles != null) {
- for (String role : roles) {
- pgi = db.getPrivileges().getRolePrivileges().get(role);
- if (pgi != null) {
- pps.putToRolePrivileges(role, pgi);
- }
- }
- }
- }
- }
- commit = true;
- return pps;
- } catch (IOException e) {
- LOG.error("Unable to get db privileges for user", e);
- throw new MetaException("Unable to get db privileges for user, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public PrincipalPrivilegeSet getTablePrivilegeSet(String dbName, String tableName,
- String userName, List<String> groupNames)
- throws InvalidObjectException, MetaException {
- boolean commit = false;
- openTransaction();
- try {
- PrincipalPrivilegeSet pps = new PrincipalPrivilegeSet();
- Table table = getHBase().getTable(dbName, tableName);
- List<PrivilegeGrantInfo> pgi;
- if (table.getPrivileges() != null) {
- if (table.getPrivileges().getUserPrivileges() != null) {
- pgi = table.getPrivileges().getUserPrivileges().get(userName);
- if (pgi != null) {
- pps.putToUserPrivileges(userName, pgi);
- }
- }
-
- if (table.getPrivileges().getRolePrivileges() != null) {
- List<String> roles = getHBase().getUserRoles(userName);
- if (roles != null) {
- for (String role : roles) {
- pgi = table.getPrivileges().getRolePrivileges().get(role);
- if (pgi != null) {
- pps.putToRolePrivileges(role, pgi);
- }
- }
- }
- }
- }
- commit = true;
- return pps;
- } catch (IOException e) {
- LOG.error("Unable to get db privileges for user", e);
- throw new MetaException("Unable to get db privileges for user, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public PrincipalPrivilegeSet getPartitionPrivilegeSet(String dbName, String tableName,
- String partition, String userName,
- List<String> groupNames) throws
- InvalidObjectException, MetaException {
- // We don't support partition privileges
- return null;
- }
-
- @Override
- public PrincipalPrivilegeSet getColumnPrivilegeSet(String dbName, String tableName,
- String partitionName, String columnName,
- String userName,
- List<String> groupNames) throws
- InvalidObjectException, MetaException {
- // We don't support column level privileges
- return null;
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalGlobalGrants(String principalName,
- PrincipalType principalType) {
- List<PrivilegeGrantInfo> grants;
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- boolean commit = false;
- openTransaction();
- try {
- PrincipalPrivilegeSet pps = getHBase().getGlobalPrivs();
- if (pps == null) return privileges;
- Map<String, List<PrivilegeGrantInfo>> map;
- switch (principalType) {
- case USER:
- map = pps.getUserPrivileges();
- break;
-
- case ROLE:
- map = pps.getRolePrivileges();
- break;
-
- default:
- throw new RuntimeException("Unknown or unsupported principal type " +
- principalType.toString());
- }
- if (map == null) return privileges;
- grants = map.get(principalName);
-
- if (grants == null || grants.size() == 0) return privileges;
- for (PrivilegeGrantInfo pgi : grants) {
- privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.GLOBAL, null,
- null, null, null), principalName, principalType, pgi));
- }
- commit = true;
- return privileges;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalDBGrants(String principalName,
- PrincipalType principalType,
- String dbName) {
- List<PrivilegeGrantInfo> grants;
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- boolean commit = false;
- openTransaction();
- try {
- Database db = getHBase().getDb(dbName);
- if (db == null) return privileges;
- PrincipalPrivilegeSet pps = db.getPrivileges();
- if (pps == null) return privileges;
- Map<String, List<PrivilegeGrantInfo>> map;
- switch (principalType) {
- case USER:
- map = pps.getUserPrivileges();
- break;
-
- case ROLE:
- map = pps.getRolePrivileges();
- break;
-
- default:
- throw new RuntimeException("Unknown or unsupported principal type " +
- principalType.toString());
- }
- if (map == null) return privileges;
- grants = map.get(principalName);
-
- if (grants == null || grants.size() == 0) return privileges;
- for (PrivilegeGrantInfo pgi : grants) {
- privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.DATABASE, dbName,
- null, null, null), principalName, principalType, pgi));
- }
- commit = true;
- return privileges;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<HiveObjectPrivilege> listAllTableGrants(String principalName,
- PrincipalType principalType,
- String dbName,
- String tableName) {
- List<PrivilegeGrantInfo> grants;
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- boolean commit = false;
- openTransaction();
- try {
- Table table = getHBase().getTable(dbName, tableName);
- if (table == null) return privileges;
- PrincipalPrivilegeSet pps = table.getPrivileges();
- if (pps == null) return privileges;
- Map<String, List<PrivilegeGrantInfo>> map;
- switch (principalType) {
- case USER:
- map = pps.getUserPrivileges();
- break;
-
- case ROLE:
- map = pps.getRolePrivileges();
- break;
-
- default:
- throw new RuntimeException("Unknown or unsupported principal type " +
- principalType.toString());
- }
- if (map == null) return privileges;
- grants = map.get(principalName);
-
- if (grants == null || grants.size() == 0) return privileges;
- for (PrivilegeGrantInfo pgi : grants) {
- privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.TABLE, dbName,
- tableName, null, null), principalName, principalType, pgi));
- }
- commit = true;
- return privileges;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalPartitionGrants(String principalName,
- PrincipalType principalType,
- String dbName,
- String tableName,
- List<String> partValues,
- String partName) {
- // We don't support partition grants
- return new ArrayList<HiveObjectPrivilege>();
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalTableColumnGrants(String principalName,
- PrincipalType principalType,
- String dbName, String tableName,
- String columnName) {
- // We don't support column grants
- return new ArrayList<HiveObjectPrivilege>();
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalPartitionColumnGrants(String principalName,
- PrincipalType principalType,
- String dbName,
- String tableName,
- List<String> partVals,
- String partName,
- String columnName) {
- // We don't support column grants
- return new ArrayList<HiveObjectPrivilege>();
- }
-
- @Override
- public boolean grantPrivileges(PrivilegeBag privileges)
- throws InvalidObjectException, MetaException, NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- for (HiveObjectPrivilege priv : privileges.getPrivileges()) {
- // Locate the right object to deal with
- PrivilegeInfo privilegeInfo = findPrivilegeToGrantOrRevoke(priv);
-
- // Now, let's see if we've already got this privilege
- for (PrivilegeGrantInfo info : privilegeInfo.grants) {
- if (info.getPrivilege().equals(priv.getGrantInfo().getPrivilege())) {
- throw new InvalidObjectException(priv.getPrincipalName() + " already has " +
- priv.getGrantInfo().getPrivilege() + " on " + privilegeInfo.typeErrMsg);
- }
- }
- privilegeInfo.grants.add(priv.getGrantInfo());
-
- writeBackGrantOrRevoke(priv, privilegeInfo);
- }
- commit = true;
- return true;
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) throws
- InvalidObjectException, MetaException, NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- for (HiveObjectPrivilege priv : privileges.getPrivileges()) {
- PrivilegeInfo privilegeInfo = findPrivilegeToGrantOrRevoke(priv);
-
- for (int i = 0; i < privilegeInfo.grants.size(); i++) {
- if (privilegeInfo.grants.get(i).getPrivilege().equals(
- priv.getGrantInfo().getPrivilege())) {
- if (grantOption) privilegeInfo.grants.get(i).setGrantOption(false);
- else privilegeInfo.grants.remove(i);
- break;
- }
- }
- writeBackGrantOrRevoke(priv, privilegeInfo);
- }
- commit = true;
- return true;
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- private static class PrivilegeInfo {
- Database db;
- Table table;
- List<PrivilegeGrantInfo> grants;
- String typeErrMsg;
- PrincipalPrivilegeSet privSet;
- }
-
- private PrivilegeInfo findPrivilegeToGrantOrRevoke(HiveObjectPrivilege privilege)
- throws MetaException, NoSuchObjectException, InvalidObjectException {
- PrivilegeInfo result = new PrivilegeInfo();
- switch (privilege.getHiveObject().getObjectType()) {
- case GLOBAL:
- try {
- result.privSet = createOnNull(getHBase().getGlobalPrivs());
- } catch (IOException e) {
- LOG.error("Unable to fetch global privileges", e);
- throw new MetaException("Unable to fetch global privileges, " + e.getMessage());
- }
- result.typeErrMsg = "global";
- break;
-
- case DATABASE:
- result.db = getDatabase(privilege.getHiveObject().getDbName());
- result.typeErrMsg = "database " + result.db.getName();
- result.privSet = createOnNull(result.db.getPrivileges());
- break;
-
- case TABLE:
- result.table = getTable(privilege.getHiveObject().getDbName(),
- privilege.getHiveObject().getObjectName());
- result.typeErrMsg = "table " + result.table.getTableName();
- result.privSet = createOnNull(result.table.getPrivileges());
- break;
-
- case PARTITION:
- case COLUMN:
- throw new RuntimeException("HBase metastore does not support partition or column " +
- "permissions");
-
- default:
- throw new RuntimeException("Woah bad, unknown object type " +
- privilege.getHiveObject().getObjectType());
- }
-
- // Locate the right PrivilegeGrantInfo
- Map<String, List<PrivilegeGrantInfo>> grantInfos;
- switch (privilege.getPrincipalType()) {
- case USER:
- grantInfos = result.privSet.getUserPrivileges();
- result.typeErrMsg = "user";
- break;
-
- case GROUP:
- throw new RuntimeException("HBase metastore does not support group permissions");
-
- case ROLE:
- grantInfos = result.privSet.getRolePrivileges();
- result.typeErrMsg = "role";
- break;
-
- default:
- throw new RuntimeException("Woah bad, unknown principal type " +
- privilege.getPrincipalType());
- }
-
- // Find the requested name in the grantInfo
- result.grants = grantInfos.get(privilege.getPrincipalName());
- if (result.grants == null) {
- // Means we don't have any grants for this user yet.
- result.grants = new ArrayList<PrivilegeGrantInfo>();
- grantInfos.put(privilege.getPrincipalName(), result.grants);
- }
- return result;
- }
-
- private PrincipalPrivilegeSet createOnNull(PrincipalPrivilegeSet pps) {
- // If this is the first time a user has been granted a privilege set will be null.
- if (pps == null) {
- pps = new PrincipalPrivilegeSet();
- }
- if (pps.getUserPrivileges() == null) {
- pps.setUserPrivileges(new HashMap<String, List<PrivilegeGrantInfo>>());
- }
- if (pps.getRolePrivileges() == null) {
- pps.setRolePrivileges(new HashMap<String, List<PrivilegeGrantInfo>>());
- }
- return pps;
- }
-
- private void writeBackGrantOrRevoke(HiveObjectPrivilege priv, PrivilegeInfo pi)
- throws MetaException, NoSuchObjectException, InvalidObjectException {
- // Now write it back
- switch (priv.getHiveObject().getObjectType()) {
- case GLOBAL:
- try {
- getHBase().putGlobalPrivs(pi.privSet);
- } catch (IOException e) {
- LOG.error("Unable to write global privileges", e);
- throw new MetaException("Unable to write global privileges, " + e.getMessage());
- }
- break;
-
- case DATABASE:
- pi.db.setPrivileges(pi.privSet);
- alterDatabase(pi.db.getName(), pi.db);
- break;
-
- case TABLE:
- pi.table.setPrivileges(pi.privSet);
- alterTable(pi.table.getDbName(), pi.table.getTableName(), pi.table);
- break;
-
- default:
- throw new RuntimeException("Dude, you missed the second switch!");
- }
- }
-
- @Override
- public Role getRole(String roleName) throws NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- Role role = getHBase().getRole(roleName);
- if (role == null) {
- throw new NoSuchObjectException("Unable to find role " + roleName);
- }
- commit = true;
- return role;
- } catch (IOException e) {
- LOG.error("Unable to get role", e);
- throw new NoSuchObjectException("Error reading table " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<String> listRoleNames() {
- boolean commit = false;
- openTransaction();
- try {
- List<Role> roles = getHBase().scanRoles();
- List<String> roleNames = new ArrayList<String>(roles.size());
- for (Role role : roles) roleNames.add(role.getRoleName());
- commit = true;
- return roleNames;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<Role> listRoles(String principalName, PrincipalType principalType) {
- List<Role> roles = new ArrayList<Role>();
- boolean commit = false;
- openTransaction();
- try {
- try {
- roles.addAll(getHBase().getPrincipalDirectRoles(principalName, principalType));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- // Add the public role if this is a user
- if (principalType == PrincipalType.USER) {
- roles.add(new Role(HiveMetaStore.PUBLIC, 0, null));
- }
- commit = true;
- return roles;
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<RolePrincipalGrant> listRolesWithGrants(String principalName,
- PrincipalType principalType) {
- boolean commit = false;
- openTransaction();
- try {
- List<Role> roles = listRoles(principalName, principalType);
- List<RolePrincipalGrant> rpgs = new ArrayList<RolePrincipalGrant>(roles.size());
- for (Role role : roles) {
- HbaseMetastoreProto.RoleGrantInfoList grants = getHBase().getRolePrincipals(role.getRoleName());
- if (grants != null) {
- for (HbaseMetastoreProto.RoleGrantInfo grant : grants.getGrantInfoList()) {
- if (grant.getPrincipalType() == HBaseUtils.convertPrincipalTypes(principalType) &&
- grant.getPrincipalName().equals(principalName)) {
- rpgs.add(new RolePrincipalGrant(role.getRoleName(), principalName, principalType,
- grant.getGrantOption(), (int) grant.getAddTime(), grant.getGrantor(),
- HBaseUtils.convertPrincipalTypes(grant.getGrantorType())));
- }
- }
- }
- }
- commit = true;
- return rpgs;
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<RolePrincipalGrant> listRoleMembers(String roleName) {
- boolean commit = false;
- openTransaction();
- try {
- HbaseMetastoreProto.RoleGrantInfoList gil = getHBase().getRolePrincipals(roleName);
- List<RolePrincipalGrant> roleMaps = new ArrayList<RolePrincipalGrant>(gil.getGrantInfoList().size());
- for (HbaseMetastoreProto.RoleGrantInfo giw : gil.getGrantInfoList()) {
- roleMaps.add(new RolePrincipalGrant(roleName, giw.getPrincipalName(),
- HBaseUtils.convertPrincipalTypes(giw.getPrincipalType()),
- giw.getGrantOption(), (int)giw.getAddTime(), giw.getGrantor(),
- HBaseUtils.convertPrincipalTypes(giw.getGrantorType())));
- }
- commit = true;
- return roleMaps;
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public Partition getPartitionWithAuth(String dbName, String tblName, List<String> partVals,
- String user_name, List<String> group_names)
- throws MetaException, NoSuchObjectException, InvalidObjectException {
- // We don't do authorization checks for partitions.
- return getPartition(dbName, tblName, partVals);
- }
-
- @Override
- public List<Partition> getPartitionsWithAuth(String dbName, String tblName, short maxParts,
- String userName, List<String> groupNames)
- throws MetaException, NoSuchObjectException, InvalidObjectException {
- // We don't do authorization checks for partitions.
- return getPartitions(dbName, tblName, maxParts);
- }
-
- @Override
- public List<String> listPartitionNamesPs(String db_name, String tbl_name, List<String> part_vals,
- short max_parts)
- throws MetaException, NoSuchObjectException {
- List<Partition> parts =
- listPartitionsPsWithAuth(db_name, tbl_name, part_vals, max_parts, null, null);
- List<String> partNames = new ArrayList<String>(parts.size());
- for (Partition part : parts) {
- partNames.add(buildExternalPartName(HiveStringUtils.normalizeIdentifier(db_name),
- HiveStringUtils.normalizeIdentifier(tbl_name), part.getValues()));
- }
- return partNames;
- }
-
-
- @Override
- public List<Partition> listPartitionsPsWithAuth(String db_name, String tbl_name,
- List<String> part_vals, short max_parts,
- String userName, List<String> groupNames)
- throws MetaException, NoSuchObjectException {
- // We don't handle auth info with partitions
- boolean commit = false;
- openTransaction();
- try {
- List<Partition> parts = getHBase().scanPartitions(HiveStringUtils.normalizeIdentifier(db_name),
- HiveStringUtils.normalizeIdentifier(tbl_name), part_vals, max_parts);
- commit = true;
- return parts;
- } catch (IOException e) {
- LOG.error("Unable to list partition names", e);
- throw new MetaException("Failed to list part names, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean updateTableColumnStatistics(ColumnStatistics colStats) throws
- NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
- boolean commit = false;
- openTransaction();
- try {
- //update table properties
- List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
- List<String> colNames = new ArrayList<>();
- for (ColumnStatisticsObj statsObj:statsObjs) {
- colNames.add(statsObj.getColName());
- }
- String dbName = colStats.getStatsDesc().getDbName();
- String tableName = colStats.getStatsDesc().getTableName();
- Table newTable = getTable(dbName, tableName);
- Table newTableCopy = newTable.deepCopy();
- StatsSetupConst.setColumnStatsState(newTableCopy.getParameters(), colNames);
- getHBase().replaceTable(newTable, newTableCopy);
-
- getHBase().updateStatistics(colStats.getStatsDesc().getDbName(),
- colStats.getStatsDesc().getTableName(), null, colStats);
-
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to update column statistics", e);
- throw new MetaException("Failed to update column statistics, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean updatePartitionColumnStatistics(ColumnStatistics colStats,
- List<String> partVals) throws
- NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
- boolean commit = false;
- openTransaction();
- try {
- // update partition properties
- String db_name = colStats.getStatsDesc().getDbName();
- String tbl_name = colStats.getStatsDesc().getTableName();
- Partition oldPart = getHBase().getPartition(db_name, tbl_name, partVals);
- Partition new_partCopy = oldPart.deepCopy();
- List<String> colNames = new ArrayList<>();
- List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
- for (ColumnStatisticsObj statsObj : statsObjs) {
- colNames.add(statsObj.getColName());
- }
- StatsSetupConst.setColumnStatsState(new_partCopy.getParameters(), colNames);
- getHBase().replacePartition(oldPart, new_partCopy,
- HBaseUtils.getPartitionKeyTypes(getTable(db_name, tbl_name).getPartitionKeys()));
-
- getHBase().updateStatistics(colStats.getStatsDesc().getDbName(),
- colStats.getStatsDesc().getTableName(), partVals, colStats);
- // We need to invalidate aggregates that include this partition
- getHBase().getStatsCache().invalidate(colStats.getStatsDesc().getDbName(),
- colStats.getStatsDesc().getTableName(), colStats.getStatsDesc().getPartName());
-
- commit = true;
- return true;
- } catch (IOException e) {
- LOG.error("Unable to update column statistics", e);
- throw new MetaException("Failed to update column statistics, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public ColumnStatistics getTableColumnStatistics(String dbName, String tableName,
- List<String> colName) throws MetaException,
- NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- ColumnStatistics cs = getHBase().getTableStatistics(dbName, tableName, colName);
- commit = true;
- return cs;
- } catch (IOException e) {
- LOG.error("Unable to fetch column statistics", e);
- throw new MetaException("Failed to fetch column statistics, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<ColumnStatistics> getPartitionColumnStatistics(String dbName, String tblName,
- List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException {
- List<List<String>> partVals = new ArrayList<List<String>>(partNames.size());
- for (String partName : partNames) {
- partVals.add(partNameToVals(partName));
- }
- boolean commit = false;
- openTransaction();
- try {
- List<ColumnStatistics> cs =
- getHBase().getPartitionStatistics(dbName, tblName, partNames, partVals, colNames);
- commit = true;
- return cs;
- } catch (IOException e) {
- LOG.error("Unable to fetch column statistics", e);
- throw new MetaException("Failed fetching column statistics, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean deletePartitionColumnStatistics(String dbName, String tableName, String partName,
- List<String> partVals, String colName) throws NoSuchObjectException, MetaException,
- InvalidObjectException, InvalidInputException {
- // NOP, stats will be deleted along with the partition when it is dropped.
- return true;
- }
-
- @Override
- public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName) throws
- NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
- // NOP, stats will be deleted along with the table when it is dropped.
- return true;
- }
-
- /**
- * Return aggregated statistics for each column in the colNames list aggregated over partitions in
- * the partNames list
- *
- */
- @Override
- public AggrStats get_aggr_stats_for(String dbName, String tblName, List<String> partNames,
- List<String> colNames) throws MetaException, NoSuchObjectException {
- List<List<String>> partVals = new ArrayList<List<String>>(partNames.size());
- for (String partName : partNames) {
- partVals.add(partNameToVals(partName));
- }
- boolean commit = false;
- boolean hasAnyStats = false;
- openTransaction();
- try {
- AggrStats aggrStats = new AggrStats();
- aggrStats.setPartsFound(0);
- for (String colName : colNames) {
- try {
- AggrStats oneCol =
- getHBase().getStatsCache().get(dbName, tblName, partNames, colName);
- if (oneCol.getColStatsSize() > 0) {
- assert oneCol.getColStatsSize() == 1;
- aggrStats.setPartsFound(oneCol.getPartsFound());
- aggrStats.addToColStats(oneCol.getColStats().get(0));
- hasAnyStats = true;
- }
- } catch (CacheLoader.InvalidCacheLoadException e) {
- LOG.debug("Found no stats for column " + colName);
- // This means we have no stats at all for this column for these partitions, so just
- // move on.
- }
- }
- commit = true;
- if (!hasAnyStats) {
- // Set the required field.
- aggrStats.setColStats(new ArrayList<ColumnStatisticsObj>());
- }
- return aggrStats;
- } catch (IOException e) {
- LOG.error("Unable to fetch aggregate column statistics", e);
- throw new MetaException("Failed fetching aggregate column statistics, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public long cleanupEvents() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean addToken(String tokenIdentifier, String delegationToken) {
- boolean commit = false;
- openTransaction();
- try {
- getHBase().putDelegationToken(tokenIdentifier, delegationToken);
- commit = true;
- return commit; // See HIVE-11302, for now always returning true
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean removeToken(String tokenIdentifier) {
- boolean commit = false;
- openTransaction();
- try {
- getHBase().deleteDelegationToken(tokenIdentifier);
- commit = true;
- return commit; // See HIVE-11302, for now always returning true
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public String getToken(String tokenIdentifier) {
- boolean commit = false;
- openTransaction();
- try {
- String token = getHBase().getDelegationToken(tokenIdentifier);
- commit = true;
- return token;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<String> getAllTokenIdentifiers() {
- boolean commit = false;
- openTransaction();
- try {
- List<String> ids = getHBase().scanDelegationTokenIdentifiers();
- commit = true;
- return ids;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public int addMasterKey(String key) throws MetaException {
- boolean commit = false;
- openTransaction();
- try {
- long seq = getHBase().getNextSequence(HBaseReadWrite.MASTER_KEY_SEQUENCE);
- getHBase().putMasterKey((int) seq, key);
- commit = true;
- return (int)seq;
- } catch (IOException e) {
- LOG.error("Unable to add master key", e);
- throw new MetaException("Failed adding master key, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public void updateMasterKey(Integer seqNo, String key) throws NoSuchObjectException,
- MetaException {
- boolean commit = false;
- openTransaction();
- try {
- if (getHBase().getMasterKey(seqNo) == null) {
- throw new NoSuchObjectException("No key found with keyId: " + seqNo);
- }
- getHBase().putMasterKey(seqNo, key);
- commit = true;
- } catch (IOException e) {
- LOG.error("Unable to update master key", e);
- throw new MetaException("Failed updating master key, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public boolean removeMasterKey(Integer keySeq) {
- boolean commit = false;
- openTransaction();
- try {
- getHBase().deleteMasterKey(keySeq);
- commit = true;
- return true; // See HIVE-11302, for now always returning true
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public String[] getMasterKeys() {
- boolean commit = false;
- openTransaction();
- try {
- List<String> keys = getHBase().scanMasterKeys();
- commit = true;
- return keys.toArray(new String[keys.size()]);
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public void verifySchema() throws MetaException {
-
- }
-
- @Override
- public String getMetaStoreSchemaVersion() throws MetaException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setMetaStoreSchemaVersion(String version, String comment) throws MetaException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void dropPartitions(String dbName, String tblName, List<String> partNames) throws
- MetaException, NoSuchObjectException {
- boolean commit = false;
- openTransaction();
- try {
- for (String partName : partNames) {
- dropPartition(dbName, tblName, partNameToVals(partName));
- }
- commit = true;
- } catch (Exception e) {
- LOG.error("Unable to drop partitions", e);
- throw new NoSuchObjectException("Failure dropping partitions, " + e.getMessage());
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalDBGrantsAll(String principalName,
- PrincipalType principalType) {
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- boolean commit = false;
- openTransaction();
- try {
- List<Database> dbs = getHBase().scanDatabases(null);
- for (Database db : dbs) {
- List<PrivilegeGrantInfo> grants;
-
- PrincipalPrivilegeSet pps = db.getPrivileges();
- if (pps == null) continue;
- Map<String, List<PrivilegeGrantInfo>> map;
- switch (principalType) {
- case USER:
- map = pps.getUserPrivileges();
- break;
-
- case ROLE:
- map = pps.getRolePrivileges();
- break;
-
- default:
- throw new RuntimeException("Unknown or unsupported principal type " +
- principalType.toString());
- }
-
- if (map == null) continue;
- grants = map.get(principalName);
- if (grants == null || grants.size() == 0) continue;
- for (PrivilegeGrantInfo pgi : grants) {
- privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.DATABASE,
- db.getName(), null, null, null), principalName, principalType, pgi));
- }
- }
- commit = true;
- return privileges;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalTableGrantsAll(String principalName,
- PrincipalType principalType) {
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- boolean commit = false;
- openTransaction();
- try {
- List<Table> tables = getHBase().scanTables(null, null);
- for (Table table : tables) {
- List<PrivilegeGrantInfo> grants;
-
- PrincipalPrivilegeSet pps = table.getPrivileges();
- if (pps == null) continue;
- Map<String, List<PrivilegeGrantInfo>> map;
- switch (principalType) {
- case USER:
- map = pps.getUserPrivileges();
- break;
-
- case ROLE:
- map = pps.getRolePrivileges();
- break;
-
- default:
- throw new RuntimeException("Unknown or unsupported principal type " +
- principalType.toString());
- }
-
- if (map == null) continue;
- grants = map.get(principalName);
- if (grants == null || grants.size() == 0) continue;
- for (PrivilegeGrantInfo pgi : grants) {
- privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.TABLE,
- table.getDbName(), table.getTableName(), null, null), principalName, principalType,
- pgi));
- }
- }
- commit = true;
- return privileges;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalPartitionGrantsAll(String principalName,
- PrincipalType principalType) {
- return new ArrayList<HiveObjectPrivilege>();
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalTableColumnGrantsAll(String principalName,
- PrincipalType principalType) {
- return new ArrayList<HiveObjectPrivilege>();
- }
-
- @Override
- public List<HiveObjectPrivilege> listPrincipalPartitionColumnGrantsAll(String principalName,
- PrincipalType principalType) {
- return new ArrayList<HiveObjectPrivilege>();
- }
-
- @Override
- public List<HiveObjectPrivilege> listGlobalGrantsAll() {
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- boolean commit = false;
- openTransaction();
- try {
- PrincipalPrivilegeSet pps = getHBase().getGlobalPrivs();
- if (pps != null) {
- for (Map.Entry<String, List<PrivilegeGrantInfo>> e : pps.getUserPrivileges().entrySet()) {
- for (PrivilegeGrantInfo pgi : e.getValue()) {
- privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.GLOBAL, null,
- null, null, null), e.getKey(), PrincipalType.USER, pgi));
- }
- }
- for (Map.Entry<String, List<PrivilegeGrantInfo>> e : pps.getRolePrivileges().entrySet()) {
- for (PrivilegeGrantInfo pgi : e.getValue()) {
- privileges.add(new HiveObjectPrivilege(new HiveObjectRef(HiveObjectType.GLOBAL, null,
- null, null, null), e.getKey(), PrincipalType.ROLE, pgi));
- }
- }
- }
- commit = true;
- return privileges;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- commitOrRoleBack(commit);
- }
- }
-
- @Override
- public List<HiveObjectPrivilege> listDBGrantsAll(String dbName) {
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- boolean commit = false;
-
<TRUNCATED>
[15/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
deleted file mode 100644
index 2cc1373..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
+++ /dev/null
@@ -1,1801 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.HiveObjectType;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.ResourceType;
-import org.apache.hadoop.hive.metastore.api.ResourceUri;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Integration tests with HBase Mini-cluster for HBaseStore
- */
-public class TestHBaseStoreIntegration extends HBaseIntegrationTests {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreIntegration.class.getName());
-
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- @BeforeClass
- public static void startup() throws Exception {
- HBaseIntegrationTests.startMiniCluster();
- }
-
- @AfterClass
- public static void shutdown() throws Exception {
- HBaseIntegrationTests.shutdownMiniCluster();
- }
-
- @Before
- public void setup() throws IOException {
- setupConnection();
- setupHBaseStore();
- }
-
- @Test
- public void createDb() throws Exception {
- String dbname = "mydb";
- Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
-
- Database d = store.getDatabase("mydb");
- Assert.assertEquals(dbname, d.getName());
- Assert.assertEquals("no description", d.getDescription());
- Assert.assertEquals("file:///tmp", d.getLocationUri());
- }
-
- @Test
- public void dropDb() throws Exception {
- String dbname = "anotherdb";
- Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
-
- Database d = store.getDatabase(dbname);
- Assert.assertNotNull(d);
-
- store.dropDatabase(dbname);
- thrown.expect(NoSuchObjectException.class);
- store.getDatabase(dbname);
- }
-
- @Test
- public void getAllDbs() throws Exception {
- String[] dbNames = new String[3];
- for (int i = 0; i < dbNames.length; i++) {
- dbNames[i] = "db" + i;
- Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
- }
-
- List<String> dbs = store.getAllDatabases();
- Assert.assertEquals(3, dbs.size());
- String[] namesFromStore = dbs.toArray(new String[3]);
- Arrays.sort(namesFromStore);
- Assert.assertArrayEquals(dbNames, namesFromStore);
- }
-
- @Test
- public void getDbsRegex() throws Exception {
- String[] dbNames = new String[3];
- for (int i = 0; i < dbNames.length; i++) {
- dbNames[i] = "db" + i;
- Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
- }
-
- List<String> dbs = store.getDatabases("db1|db2");
- Assert.assertEquals(2, dbs.size());
- String[] namesFromStore = dbs.toArray(new String[2]);
- Arrays.sort(namesFromStore);
- Assert.assertArrayEquals(Arrays.copyOfRange(dbNames, 1, 3), namesFromStore);
-
- dbs = store.getDatabases("db*");
- Assert.assertEquals(3, dbs.size());
- namesFromStore = dbs.toArray(new String[3]);
- Arrays.sort(namesFromStore);
- Assert.assertArrayEquals(dbNames, namesFromStore);
- }
-
- @Test
- public void getFuncsRegex() throws Exception {
- String dbname = "default";
- int now = (int)(System.currentTimeMillis()/1000);
- String[] funcNames = new String[3];
- for (int i = 0; i < funcNames.length; i++) {
- funcNames[i] = "func" + i;
- store.createFunction(new Function(funcNames[i], dbname, "o.a.h.h.myfunc", "me",
- PrincipalType.USER, now, FunctionType.JAVA,
- Arrays.asList(new ResourceUri(ResourceType.JAR,
- "file:/tmp/somewhere"))));
- }
-
- List<String> funcs = store.getFunctions(dbname, "func1|func2");
- Assert.assertEquals(2, funcs.size());
- String[] namesFromStore = funcs.toArray(new String[2]);
- Arrays.sort(namesFromStore);
- Assert.assertArrayEquals(Arrays.copyOfRange(funcNames, 1, 3), namesFromStore);
-
- funcs = store.getFunctions(dbname, "func*");
- Assert.assertEquals(3, funcs.size());
- namesFromStore = funcs.toArray(new String[3]);
- Arrays.sort(namesFromStore);
- Assert.assertArrayEquals(funcNames, namesFromStore);
-
- funcs = store.getFunctions("nosuchdb", "func*");
- Assert.assertEquals(0, funcs.size());
- }
-
- @Test
- public void createTable() throws Exception {
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table("mytable", "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- Table t = store.getTable("default", "mytable");
- Assert.assertEquals(1, t.getSd().getColsSize());
- Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
- Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp", t.getSd().getLocation());
- Assert.assertEquals("input", t.getSd().getInputFormat());
- Assert.assertEquals("output", t.getSd().getOutputFormat());
- Assert.assertEquals("me", t.getOwner());
- Assert.assertEquals("default", t.getDbName());
- Assert.assertEquals("mytable", t.getTableName());
- }
-
- @Test
- public void alterTable() throws Exception {
- String tableName = "alttable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- startTime += 10;
- table.setLastAccessTime(startTime);
- LOG.debug("XXX alter table test");
- store.alterTable("default", tableName, table);
-
- Table t = store.getTable("default", tableName);
- LOG.debug("Alter table time " + t.getLastAccessTime());
- Assert.assertEquals(1, t.getSd().getColsSize());
- Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
- Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp", t.getSd().getLocation());
- Assert.assertEquals("input", t.getSd().getInputFormat());
- Assert.assertEquals("output", t.getSd().getOutputFormat());
- Assert.assertEquals("me", t.getOwner());
- Assert.assertEquals("default", t.getDbName());
- Assert.assertEquals(tableName, t.getTableName());
- Assert.assertEquals(startTime, t.getLastAccessTime());
- }
-
- @Test
- public void getAllTables() throws Exception {
- String dbNames[] = new String[]{"db0", "db1"}; // named to match getAllDbs so we get the
- // right number of databases in that test.
- String tableNames[] = new String[]{"curly", "larry", "moe"};
-
- for (int i = 0; i < dbNames.length; i++) {
- store.createDatabase(new Database(dbNames[i], "no description", "file:///tmp",
- emptyParameters));
- }
-
- for (int i = 0; i < dbNames.length; i++) {
- for (int j = 0; j < tableNames.length; j++) {
- int startTime = (int) (System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false,
- 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableNames[j], dbNames[i], "me", startTime, startTime, 0, sd,
- null,
- emptyParameters, null, null, null);
- store.createTable(table);
- }
- }
-
- List<String> fetchedNames = store.getAllTables(dbNames[0]);
- Assert.assertEquals(3, fetchedNames.size());
- String[] sortedFetchedNames = fetchedNames.toArray(new String[fetchedNames.size()]);
- Arrays.sort(sortedFetchedNames);
- Assert.assertArrayEquals(tableNames, sortedFetchedNames);
-
- List<String> regexNames = store.getTables(dbNames[0], "*y");
- Assert.assertEquals(2, regexNames.size());
- String[] sortedRegexNames = regexNames.toArray(new String[regexNames.size()]);
- Arrays.sort(sortedRegexNames);
- Assert.assertArrayEquals(Arrays.copyOfRange(tableNames, 0, 2), sortedRegexNames);
-
- List<Table> fetchedTables = store.getTableObjectsByName(dbNames[1],
- Arrays.asList(Arrays.copyOfRange(tableNames, 1, 3)));
- Assert.assertEquals(2, fetchedTables.size());
- sortedFetchedNames = new String[fetchedTables.size()];
- for (int i = 0; i < fetchedTables.size(); i++) {
- sortedFetchedNames[i] = fetchedTables.get(i).getTableName();
- }
- Arrays.sort(sortedFetchedNames);
- Assert.assertArrayEquals(Arrays.copyOfRange(tableNames, 1, 3), sortedFetchedNames);
- }
-
- @Test
- public void dropTable() throws Exception {
- String tableName = "dtable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- Table t = store.getTable("default", tableName);
- Assert.assertNotNull(t);
-
- store.dropTable("default", tableName);
- Assert.assertNull(store.getTable("default", tableName));
- }
-
- @Test
- public void createPartition() throws Exception {
- String dbName = "default";
- String tableName = "myparttable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> vals = new ArrayList<String>();
- vals.add("fred");
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=fred");
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Partition p = store.getPartition(dbName, tableName, vals);
- Assert.assertEquals(1, p.getSd().getColsSize());
- Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
- Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
- Assert.assertEquals("input", p.getSd().getInputFormat());
- Assert.assertEquals("output", p.getSd().getOutputFormat());
- Assert.assertEquals(dbName, p.getDbName());
- Assert.assertEquals(tableName, p.getTableName());
- Assert.assertEquals(1, p.getValuesSize());
- Assert.assertEquals("fred", p.getValues().get(0));
- }
-
- @Test
- public void addPartitions() throws Exception {
- String dbName = "default";
- String tableName = "addParts";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
- List<Partition> partitions = new ArrayList<Partition>();
- for (String val : partVals) {
- List<String> vals = new ArrayList<String>();
- vals.add(val);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=" + val);
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- partitions.add(part);
- }
- store.addPartitions(dbName, tableName, partitions);
-
- List<String> partNames = store.listPartitionNames(dbName, tableName, (short) -1);
- Assert.assertEquals(5, partNames.size());
- String[] names = partNames.toArray(new String[partNames.size()]);
- Arrays.sort(names);
- String[] canonicalNames = partVals.toArray(new String[partVals.size()]);
- for (int i = 0; i < canonicalNames.length; i++) canonicalNames[i] = "pc=" + canonicalNames[i];
- Assert.assertArrayEquals(canonicalNames, names);
- }
-
- @Test
- public void alterPartitions() throws Exception {
- String dbName = "default";
- String tableName = "alterParts";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
- List<Partition> partitions = new ArrayList<Partition>();
- List<List<String>> allVals = new ArrayList<List<String>>();
- for (String val : partVals) {
- List<String> vals = new ArrayList<String>();
- allVals.add(vals);
- vals.add(val);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=" + val);
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- partitions.add(part);
- }
- store.addPartitions(dbName, tableName, partitions);
-
- for (Partition p : partitions) p.setLastAccessTime(startTime + 10);
- store.alterPartitions(dbName, tableName, allVals, partitions);
-
- partitions = store.getPartitions(dbName, tableName, -1);
- for (Partition part : partitions) {
- Assert.assertEquals(startTime + 10, part.getLastAccessTime());
- }
- }
-
- @Test
- public void getPartitions() throws Exception {
- String dbName = "default";
- String tableName = "manyParts";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
- for (String val : partVals) {
- List<String> vals = new ArrayList<String>();
- vals.add(val);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=" + val);
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Partition p = store.getPartition(dbName, tableName, vals);
- Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
- }
-
- List<Partition> parts = store.getPartitions(dbName, tableName, -1);
- Assert.assertEquals(5, parts.size());
- String[] pv = new String[5];
- for (int i = 0; i < 5; i++) pv[i] = parts.get(i).getValues().get(0);
- Arrays.sort(pv);
- Assert.assertArrayEquals(pv, partVals.toArray(new String[5]));
- }
-
- @Test
- public void listPartitions() throws Exception {
- String dbName = "default";
- String tableName = "listParts";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- partCols.add(new FieldSchema("region", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- String[][] partVals = new String[][]{{"today", "north america"}, {"tomorrow", "europe"}};
- for (String[] pv : partVals) {
- List<String> vals = new ArrayList<String>();
- for (String v : pv) vals.add(v);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=" + pv[0] + "/region=" + pv[1]);
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
- }
-
- List<String> names = store.listPartitionNames(dbName, tableName, (short) -1);
- Assert.assertEquals(2, names.size());
- String[] resultNames = names.toArray(new String[names.size()]);
- Arrays.sort(resultNames);
- Assert.assertArrayEquals(resultNames, new String[]{"pc=today/region=north america",
- "pc=tomorrow/region=europe"});
-
- List<Partition> parts = store.getPartitionsByNames(dbName, tableName, names);
- Assert.assertArrayEquals(partVals[0], parts.get(0).getValues().toArray(new String[2]));
- Assert.assertArrayEquals(partVals[1], parts.get(1).getValues().toArray(new String[2]));
-
- store.dropPartitions(dbName, tableName, names);
- List<Partition> afterDropParts = store.getPartitions(dbName, tableName, -1);
- Assert.assertEquals(0, afterDropParts.size());
- }
-
- @Test
- public void listPartitionsWithPs() throws Exception {
- String dbName = "default";
- String tableName = "listPartitionsWithPs";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("ds", "string", ""));
- partCols.add(new FieldSchema("region", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- String[][] partVals = new String[][]{{"today", "north america"}, {"today", "europe"},
- {"tomorrow", "north america"}, {"tomorrow", "europe"}};
- for (String[] pv : partVals) {
- List<String> vals = new ArrayList<String>();
- for (String v : pv) vals.add(v);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]);
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
- }
-
- // We only test listPartitionNamesPs since it calls listPartitionsPsWithAuth anyway.
- // Test the case where we completely specify the partition
- List<String> partitionNames =
- store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0]), (short) -1);
- Assert.assertEquals(1, partitionNames.size());
- Assert.assertEquals("ds=today/region=north america", partitionNames.get(0));
-
- // Leave off the last value of the partition
- partitionNames =
- store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0][0]), (short)-1);
- Assert.assertEquals(2, partitionNames.size());
- String[] names = partitionNames.toArray(new String[partitionNames.size()]);
- Arrays.sort(names);
- Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
- "ds=today/region=north america"}, names);
-
- // Put a star in the last value of the partition
- partitionNames =
- store.listPartitionNamesPs(dbName, tableName, Arrays.asList("today", "*"), (short)-1);
- Assert.assertEquals(2, partitionNames.size());
- names = partitionNames.toArray(new String[partitionNames.size()]);
- Arrays.sort(names);
- Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
- "ds=today/region=north america"}, names);
-
- // Put a star in the first value of the partition
- partitionNames =
- store.listPartitionNamesPs(dbName, tableName, Arrays.asList("*", "europe"), (short)-1);
- Assert.assertEquals(2, partitionNames.size());
- names = partitionNames.toArray(new String[partitionNames.size()]);
- Arrays.sort(names);
- Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
- "ds=tomorrow/region=europe"}, names);
- }
-
-
- @Test
- public void getPartitionsByFilter() throws Exception {
- String dbName = "default";
- String tableName = "getPartitionsByFilter";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("ds", "string", ""));
- partCols.add(new FieldSchema("region", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- String[][] partVals = new String[][]{{"20010101", "north america"}, {"20010101", "europe"},
- {"20010102", "north america"}, {"20010102", "europe"}, {"20010103", "north america"}};
- for (String[] pv : partVals) {
- List<String> vals = new ArrayList<String>();
- for (String v : pv) vals.add(v);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]);
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
- }
-
- // We only test getPartitionsByFilter since it calls same code as getPartitionsByExpr anyway.
- // Test the case where we completely specify the partition
- List<Partition> parts = null;
- parts = store.getPartitionsByFilter(dbName, tableName, "ds > '20010101'", (short) -1);
- checkPartVals(parts, "[20010102, north america]", "[20010102, europe]",
- "[20010103, north america]");
-
- parts = store.getPartitionsByFilter(dbName, tableName, "ds >= '20010102'", (short) -1);
- checkPartVals(parts, "[20010102, north america]", "[20010102, europe]",
- "[20010103, north america]");
-
- parts = store.getPartitionsByFilter(dbName, tableName,
- "ds >= '20010102' and region = 'europe' ", (short) -1);
- // filtering on first partition is only implemented as of now, so it will
- // not filter on region
- checkPartVals(parts, "[20010102, north america]", "[20010102, europe]",
- "[20010103, north america]");
-
- parts = store.getPartitionsByFilter(dbName, tableName,
- "ds >= '20010101' and ds < '20010102'", (short) -1);
- checkPartVals(parts,"[20010101, north america]", "[20010101, europe]");
-
- parts = store.getPartitionsByFilter(dbName, tableName,
- "ds = '20010102' or ds < '20010103'", (short) -1);
- checkPartVals(parts, "[20010101, north america]", "[20010101, europe]",
- "[20010102, north america]", "[20010102, europe]");
-
- // test conversion to DNF
- parts = store.getPartitionsByFilter(dbName, tableName,
- "ds = '20010102' and (ds = '20010102' or region = 'europe')", (short) -1);
- // filtering on first partition is only implemented as of now, so it will not filter on region
- checkPartVals(parts, "[20010102, north america]", "[20010102, europe]");
-
- parts = store.getPartitionsByFilter(dbName, tableName,
- "region = 'europe'", (short) -1);
- // filtering on first partition is only implemented as of now, so it will not filter on region
- checkPartVals(parts, "[20010101, north america]", "[20010101, europe]",
- "[20010102, north america]", "[20010102, europe]", "[20010103, north america]");
-
- }
-
- /**
- * Check if the given partitions have same values as given partitions value strings
- * @param parts given partitions
- * @param expectedPartVals
- */
- private void checkPartVals(List<Partition> parts, String ... expectedPartVals) {
- Assert.assertEquals("number of partitions", expectedPartVals.length, parts.size());
- Set<String> partValStrings = new TreeSet<String>();
- for(Partition part : parts) {
- partValStrings.add(part.getValues().toString());
- }
- partValStrings.equals(new TreeSet(Arrays.asList(expectedPartVals)));
- }
-
- @Test
- public void dropPartition() throws Exception {
- String dbName = "default";
- String tableName = "myparttable2";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> vals = Arrays.asList("fred");
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=fred");
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Assert.assertNotNull(store.getPartition(dbName, tableName, vals));
- store.dropPartition(dbName, tableName, vals);
- thrown.expect(NoSuchObjectException.class);
- store.getPartition(dbName, tableName, vals);
- }
-
- @Test
- public void createRole() throws Exception {
- int now = (int)System.currentTimeMillis()/1000;
- String roleName = "myrole";
- store.addRole(roleName, "me");
-
- Role r = store.getRole(roleName);
- Assert.assertEquals(roleName, r.getRoleName());
- Assert.assertEquals("me", r.getOwnerName());
- Assert.assertTrue(now <= r.getCreateTime());
- }
-
- @Test
- public void dropRole() throws Exception {
- String roleName = "anotherrole";
- store.addRole(roleName, "me");
-
- Role r = store.getRole(roleName);
- Assert.assertEquals(roleName, r.getRoleName());
-
- store.removeRole(roleName);
- thrown.expect(NoSuchObjectException.class);
- store.getRole(roleName);
- }
-
- @Test
- public void grantRevokeRoles() throws Exception {
- int now = (int)(System.currentTimeMillis()/1000);
- String roleName1 = "role1";
- store.addRole(roleName1, "me");
- String roleName2 = "role2";
- store.addRole(roleName2, "me");
-
- Role role1 = store.getRole(roleName1);
- Role role2 = store.getRole(roleName2);
-
- store.grantRole(role1, "fred", PrincipalType.USER, "bob", PrincipalType.USER, false);
- store.grantRole(role2, roleName1, PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
- store.grantRole(role2, "fred", PrincipalType.USER, "admin", PrincipalType.ROLE, false);
-
- List<Role> roles = store.listRoles("fred", PrincipalType.USER);
- Assert.assertEquals(3, roles.size());
- boolean sawRole1 = false, sawRole2 = false, sawPublic = false;
- for (Role role : roles) {
- if (role.getRoleName().equals(roleName1)) {
- sawRole1 = true;
- } else if (role.getRoleName().equals(roleName2)) {
- sawRole2 = true;
- } else if (role.getRoleName().equals(HiveMetaStore.PUBLIC)) {
- sawPublic = true;
- } else {
- Assert.fail("Unknown role name " + role.getRoleName());
- }
- }
- Assert.assertTrue(sawRole1 && sawRole2 && sawPublic);
-
- roles = store.listRoles("fred", PrincipalType.ROLE);
- Assert.assertEquals(0, roles.size());
-
- roles = store.listRoles(roleName1, PrincipalType.ROLE);
- Assert.assertEquals(1, roles.size());
- Role role = roles.get(0);
- Assert.assertEquals(roleName2, role.getRoleName());
-
- // Test listing all members in a role
- List<RolePrincipalGrant> grants = store.listRoleMembers(roleName1);
- Assert.assertEquals(1, grants.size());
- Assert.assertEquals("fred", grants.get(0).getPrincipalName());
- Assert.assertEquals(PrincipalType.USER, grants.get(0).getPrincipalType());
- Assert.assertTrue("Expected grant time of " + now + " got " + grants.get(0).getGrantTime(),
- grants.get(0).getGrantTime() >= now);
- Assert.assertEquals("bob", grants.get(0).getGrantorName());
- Assert.assertEquals(PrincipalType.USER, grants.get(0).getGrantorPrincipalType());
- Assert.assertFalse(grants.get(0).isGrantOption());
-
- grants = store.listRoleMembers(roleName2);
- Assert.assertEquals(2, grants.size());
- boolean sawFred = false;
- sawRole1 = false;
- for (RolePrincipalGrant m : grants) {
- if ("fred".equals(m.getPrincipalName())) sawFred = true;
- else if (roleName1.equals(m.getPrincipalName())) sawRole1 = true;
- else Assert.fail("Unexpected principal " + m.getPrincipalName());
- }
- Assert.assertTrue(sawFred && sawRole1);
-
- // Revoke a role with grant option, make sure it just goes to no grant option
- store.revokeRole(role2, roleName1, PrincipalType.ROLE, true);
- roles = store.listRoles(roleName1, PrincipalType.ROLE);
- Assert.assertEquals(1, roles.size());
- Assert.assertEquals(roleName2, roles.get(0).getRoleName());
-
- grants = store.listRoleMembers(roleName1);
- Assert.assertFalse(grants.get(0).isGrantOption());
-
- // Drop a role, make sure it is properly removed from the map
- store.removeRole(roleName1);
- roles = store.listRoles("fred", PrincipalType.USER);
- Assert.assertEquals(2, roles.size());
- sawRole2 = sawPublic = false;
- for (Role m : roles) {
- if (m.getRoleName().equals(roleName2)) sawRole2 = true;
- else if (m.getRoleName().equals(HiveMetaStore.PUBLIC)) sawPublic = true;
- else Assert.fail("Unknown role " + m.getRoleName());
- }
- Assert.assertTrue(sawRole2 && sawPublic);
- roles = store.listRoles(roleName1, PrincipalType.ROLE);
- Assert.assertEquals(0, roles.size());
-
- // Revoke a role without grant option, make sure it goes away
- store.revokeRole(role2, "fred", PrincipalType.USER, false);
- roles = store.listRoles("fred", PrincipalType.USER);
- Assert.assertEquals(1, roles.size());
- Assert.assertEquals(HiveMetaStore.PUBLIC, roles.get(0).getRoleName());
- }
-
- @Test
- public void userToRoleMap() throws Exception {
- String roleName1 = "utrm1";
- store.addRole(roleName1, "me");
- String roleName2 = "utrm2";
- store.addRole(roleName2, "me");
- String user1 = "wilma";
- String user2 = "betty";
-
- Role role1 = store.getRole(roleName1);
- Role role2 = store.getRole(roleName2);
-
- store.grantRole(role1, user1, PrincipalType.USER, "bob", PrincipalType.USER, false);
- store.grantRole(role1, roleName2, PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
-
- List<String> roles = HBaseReadWrite.getInstance().getUserRoles(user1);
- Assert.assertEquals(2, roles.size());
- String[] roleNames = roles.toArray(new String[roles.size()]);
- Arrays.sort(roleNames);
- Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames);
-
- store.grantRole(role2, user1, PrincipalType.USER, "admin", PrincipalType.ROLE, false);
- store.grantRole(role1, user2, PrincipalType.USER, "bob", PrincipalType.USER, false);
-
- HBaseReadWrite.setConf(conf);
- roles = HBaseReadWrite.getInstance().getUserRoles(user2);
- Assert.assertEquals(2, roles.size());
- roleNames = roles.toArray(new String[roles.size()]);
- Arrays.sort(roleNames);
- Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames);
-
- store.revokeRole(role1, roleName2, PrincipalType.ROLE, false);
-
- // user1 should still have both roles since she was granted into role1 specifically. user2
- // should only have role2 now since role2 was revoked from role1.
- roles = HBaseReadWrite.getInstance().getUserRoles(user1);
- Assert.assertEquals(2, roles.size());
- roleNames = roles.toArray(new String[roles.size()]);
- Arrays.sort(roleNames);
- Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames);
-
- roles = HBaseReadWrite.getInstance().getUserRoles(user2);
- Assert.assertEquals(1, roles.size());
- Assert.assertEquals(roleName1, roles.get(0));
- }
-
- @Test
- public void userToRoleMapOnDrop() throws Exception {
- String roleName1 = "utrmod1";
- store.addRole(roleName1, "me");
- String roleName2 = "utrmod2";
- store.addRole(roleName2, "me");
- String user1 = "pebbles";
- String user2 = "bam-bam";
-
- Role role1 = store.getRole(roleName1);
- Role role2 = store.getRole(roleName2);
-
- store.grantRole(role1, user1, PrincipalType.USER, "bob", PrincipalType.USER, false);
- store.grantRole(role1, roleName2, PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
- store.grantRole(role1, user2, PrincipalType.USER, "bob", PrincipalType.USER, false);
-
- List<String> roles = HBaseReadWrite.getInstance().getUserRoles(user2);
- Assert.assertEquals(2, roles.size());
- String[] roleNames = roles.toArray(new String[roles.size()]);
- Arrays.sort(roleNames);
- Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames);
-
- store.removeRole(roleName2);
-
- HBaseReadWrite.setConf(conf);
- roles = HBaseReadWrite.getInstance().getUserRoles(user1);
- Assert.assertEquals(1, roles.size());
- Assert.assertEquals(roleName1, roles.get(0));
-
- roles = HBaseReadWrite.getInstance().getUserRoles(user2);
- Assert.assertEquals(1, roles.size());
- Assert.assertEquals(roleName1, roles.get(0));
- }
-
- @Test
- public void grantRevokeGlobalPrivileges() throws Exception {
- doGrantRevoke(HiveObjectType.GLOBAL, null, null, new String[] {"grpg1", "grpg2"},
- new String[] {"bugs", "elmer", "daphy", "wiley"});
- }
-
- @Test
- public void grantRevokeDbPrivileges() throws Exception {
- String dbName = "grdbp_db";
- try {
- Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
- doGrantRevoke(HiveObjectType.DATABASE, dbName, null,
- new String[] {"grdbp_role1", "grdbp_role2"},
- new String[] {"fred", "barney", "wilma", "betty"});
- } finally {
- store.dropDatabase(dbName);
- }
- }
-
- @Test
- public void grantRevokeTablePrivileges() throws Exception {
- String dbName = "grtp_db";
- String tableName = "grtp_table";
- try {
- Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
- doGrantRevoke(HiveObjectType.TABLE, dbName, tableName,
- new String[] {"grtp_role1", "grtp_role2"},
- new String[] {"batman", "robin", "superman", "wonderwoman"});
-
- } finally {
- if (store.getTable(dbName, tableName) != null) store.dropTable(dbName, tableName);
- store.dropDatabase(dbName);
- }
- }
-
- private void doGrantRevoke(HiveObjectType objectType, String dbName, String tableName,
- String[] roleNames, String[] userNames)
- throws Exception {
- store.addRole(roleNames[0], "me");
- store.addRole(roleNames[1], "me");
- int now = (int)(System.currentTimeMillis() / 1000);
-
- Role role1 = store.getRole(roleNames[0]);
- Role role2 = store.getRole(roleNames[1]);
- store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false);
- store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
- store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false);
-
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- HiveObjectRef hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
- PrivilegeGrantInfo grantInfo =
- new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false);
- HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER,
- grantInfo);
- privileges.add(hop);
-
- hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
- grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
- hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
- privileges.add(hop);
-
- hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
- grantInfo = new PrivilegeGrantInfo("exec", now, "me", PrincipalType.USER, false);
- hop = new HiveObjectPrivilege(hiveObjRef, roleNames[1], PrincipalType.ROLE, grantInfo);
- privileges.add(hop);
-
- hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
- grantInfo = new PrivilegeGrantInfo("create", now, "me", PrincipalType.USER, true);
- hop = new HiveObjectPrivilege(hiveObjRef, userNames[2], PrincipalType.USER, grantInfo);
- privileges.add(hop);
-
- hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
- grantInfo = new PrivilegeGrantInfo("create2", now, "me", PrincipalType.USER, true);
- hop = new HiveObjectPrivilege(hiveObjRef, userNames[2], PrincipalType.USER, grantInfo);
- privileges.add(hop);
-
- PrivilegeBag pBag = new PrivilegeBag(privileges);
- store.grantPrivileges(pBag);
-
- PrincipalPrivilegeSet pps = getPPS(objectType, dbName, tableName, userNames[0]);
-
- Assert.assertEquals(1, pps.getUserPrivilegesSize());
- Assert.assertEquals(1, pps.getUserPrivileges().get(userNames[0]).size());
- grantInfo = pps.getUserPrivileges().get(userNames[0]).get(0);
- Assert.assertEquals("read", grantInfo.getPrivilege());
- Assert.assertTrue(now <= grantInfo.getCreateTime());
- Assert.assertEquals("me", grantInfo.getGrantor());
- Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
- Assert.assertFalse(grantInfo.isGrantOption());
-
- Assert.assertEquals(2, pps.getRolePrivilegesSize());
- Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[0]).size());
- grantInfo = pps.getRolePrivileges().get(roleNames[0]).get(0);
- Assert.assertEquals("write", grantInfo.getPrivilege());
- Assert.assertTrue(now <= grantInfo.getCreateTime());
- Assert.assertEquals("me", grantInfo.getGrantor());
- Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
- Assert.assertTrue(grantInfo.isGrantOption());
-
- Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[1]).size());
- grantInfo = pps.getRolePrivileges().get(roleNames[1]).get(0);
- Assert.assertEquals("exec", grantInfo.getPrivilege());
- Assert.assertTrue(now <= grantInfo.getCreateTime());
- Assert.assertEquals("me", grantInfo.getGrantor());
- Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
- Assert.assertFalse(grantInfo.isGrantOption());
-
- pps = getPPS(objectType, dbName, tableName, userNames[1]);
-
- Assert.assertEquals(0, pps.getUserPrivilegesSize());
-
- Assert.assertEquals(1, pps.getRolePrivilegesSize());
- Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[1]).size());
- grantInfo = pps.getRolePrivileges().get(roleNames[1]).get(0);
- Assert.assertEquals("exec", grantInfo.getPrivilege());
- Assert.assertTrue(now <= grantInfo.getCreateTime());
- Assert.assertEquals("me", grantInfo.getGrantor());
- Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
- Assert.assertFalse(grantInfo.isGrantOption());
-
- pps = getPPS(objectType, dbName, tableName, userNames[2]);
-
- Assert.assertEquals(1, pps.getUserPrivilegesSize());
- Assert.assertEquals(2, pps.getUserPrivileges().get(userNames[2]).size());
- Assert.assertEquals(0, pps.getRolePrivilegesSize());
-
- pps = getPPS(objectType, dbName, tableName, userNames[3]);
- Assert.assertEquals(0, pps.getUserPrivilegesSize());
- Assert.assertEquals(0, pps.getRolePrivilegesSize());
-
- // Test that removing role removes the role grants
- store.removeRole(roleNames[1]);
- checkRoleRemovedFromAllPrivileges(objectType, dbName, tableName, roleNames[1]);
- pps = getPPS(objectType, dbName, tableName, userNames[0]);
-
- Assert.assertEquals(1, pps.getRolePrivilegesSize());
- Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[0]).size());
-
- pps = getPPS(objectType, dbName, tableName, userNames[1]);
-
- Assert.assertEquals(0, pps.getRolePrivilegesSize());
-
- // Test that revoking with grant option = true just removes grant option
- privileges.clear();
- hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
- grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
- hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
- privileges.add(hop);
-
- hiveObjRef = new HiveObjectRef(objectType, dbName, tableName, null, null);
- grantInfo = new PrivilegeGrantInfo("create2", now, "me", PrincipalType.USER, true);
- hop = new HiveObjectPrivilege(hiveObjRef, userNames[2], PrincipalType.USER, grantInfo);
- privileges.add(hop);
-
- pBag = new PrivilegeBag(privileges);
- store.revokePrivileges(pBag, true);
- pps = getPPS(objectType, dbName, tableName, userNames[0]);
-
- Assert.assertEquals(1, pps.getRolePrivilegesSize());
- Assert.assertEquals(1, pps.getRolePrivileges().get(roleNames[0]).size());
- grantInfo = pps.getRolePrivileges().get(roleNames[0]).get(0);
- Assert.assertEquals("write", grantInfo.getPrivilege());
- Assert.assertTrue(now <= grantInfo.getCreateTime());
- Assert.assertEquals("me", grantInfo.getGrantor());
- Assert.assertEquals(PrincipalType.USER, grantInfo.getGrantorType());
- Assert.assertFalse(grantInfo.isGrantOption());
-
- pps = getPPS(objectType, dbName, tableName, userNames[2]);
-
- Assert.assertEquals(1, pps.getUserPrivilegesSize());
- Assert.assertEquals(2, pps.getUserPrivileges().get(userNames[2]).size());
- for (PrivilegeGrantInfo pgi : pps.getUserPrivileges().get(userNames[2])) {
- if (pgi.getPrivilege().equals("create")) Assert.assertTrue(pgi.isGrantOption());
- else if (pgi.getPrivilege().equals("create2")) Assert.assertFalse(pgi.isGrantOption());
- else Assert.fail("huh?");
- }
-
- // Test revoking revokes
- store.revokePrivileges(pBag, false);
-
- pps = getPPS(objectType, dbName, tableName, userNames[0]);
-
- Assert.assertEquals(1, pps.getUserPrivilegesSize());
- Assert.assertEquals(1, pps.getRolePrivilegesSize());
- Assert.assertEquals(0, pps.getRolePrivileges().get(roleNames[0]).size());
-
- pps = getPPS(objectType, dbName, tableName, userNames[2]);
- Assert.assertEquals(1, pps.getUserPrivilegesSize());
- Assert.assertEquals(1, pps.getUserPrivileges().get(userNames[2]).size());
- Assert.assertEquals("create", pps.getUserPrivileges().get(userNames[2]).get(0).getPrivilege());
- Assert.assertEquals(0, pps.getRolePrivilegesSize());
- }
-
- private PrincipalPrivilegeSet getPPS(HiveObjectType objectType, String dbName, String tableName,
- String userName)
- throws InvalidObjectException, MetaException {
- switch (objectType) {
- case GLOBAL: return store.getUserPrivilegeSet(userName, null);
- case DATABASE: return store.getDBPrivilegeSet(dbName, userName, null);
- case TABLE: return store.getTablePrivilegeSet(dbName, tableName, userName, null);
- default: throw new RuntimeException("huh?");
- }
- }
-
- private void checkRoleRemovedFromAllPrivileges(HiveObjectType objectType, String dbName,
- String tableName, String roleName)
- throws IOException, NoSuchObjectException, MetaException {
- List<PrivilegeGrantInfo> pgi = null;
- switch (objectType) {
- case GLOBAL:
- pgi = HBaseReadWrite.getInstance().getGlobalPrivs().getRolePrivileges().get(roleName);
- break;
-
- case DATABASE:
- pgi = store.getDatabase(dbName).getPrivileges().getRolePrivileges().get(roleName);
- break;
-
- case TABLE:
- pgi = store.getTable(dbName, tableName).getPrivileges().getRolePrivileges().get(roleName);
- break;
-
- default:
- Assert.fail();
- }
-
- Assert.assertNull("Expected null for role " + roleName + " for type " + objectType.toString()
- + " with db " + dbName + " and table " + tableName, pgi);
- }
-
- @Test
- public void listDbGrants() throws Exception {
- String dbNames[] = new String[] {"ldbg_db1", "ldbg_db2"};
- try {
- Database db = new Database(dbNames[0], "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
- db = new Database(dbNames[1], "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
- String[] roleNames = new String[]{"ldbg_role1", "ldbg_role2"};
- String[] userNames = new String[]{"frodo", "sam"};
-
- store.addRole(roleNames[0], "me");
- store.addRole(roleNames[1], "me");
- int now = (int)(System.currentTimeMillis() / 1000);
-
- Role role1 = store.getRole(roleNames[0]);
- Role role2 = store.getRole(roleNames[1]);
- store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false);
- store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
- store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false);
-
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- HiveObjectRef hiveObjRef =
- new HiveObjectRef(HiveObjectType.DATABASE, dbNames[0], null, null, null);
- PrivilegeGrantInfo grantInfo =
- new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false);
- HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER,
- grantInfo);
- privileges.add(hop);
-
- grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
- hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
- privileges.add(hop);
-
- PrivilegeBag pBag = new PrivilegeBag(privileges);
- store.grantPrivileges(pBag);
-
- List<HiveObjectPrivilege> hops =
- store.listPrincipalDBGrants(roleNames[0], PrincipalType.ROLE, dbNames[0]);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listPrincipalDBGrants(userNames[0], PrincipalType.USER, dbNames[0]);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listPrincipalDBGrants(roleNames[1], PrincipalType.ROLE, dbNames[0]);
- Assert.assertEquals(0, hops.size());
- hops = store.listPrincipalDBGrants(userNames[1], PrincipalType.USER, dbNames[0]);
- Assert.assertEquals(0, hops.size());
-
- hops = store.listPrincipalDBGrants(roleNames[0], PrincipalType.ROLE, dbNames[1]);
- Assert.assertEquals(0, hops.size());
- hops = store.listPrincipalDBGrants(userNames[0], PrincipalType.USER, dbNames[1]);
- Assert.assertEquals(0, hops.size());
-
- hops = store.listDBGrantsAll(dbNames[0]);
- Assert.assertEquals(2, hops.size());
- boolean sawUser = false, sawRole = false;
- for (HiveObjectPrivilege h : hops) {
- if (h.getPrincipalName().equals(userNames[0])) {
- Assert.assertEquals(PrincipalType.USER, h.getPrincipalType());
- Assert.assertEquals(HiveObjectType.DATABASE, h.getHiveObject().getObjectType());
- Assert.assertEquals("read", h.getGrantInfo().getPrivilege());
- sawUser = true;
- } else if (h.getPrincipalName().equals(roleNames[0])) {
- Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType());
- Assert.assertEquals(HiveObjectType.DATABASE, h.getHiveObject().getObjectType());
- Assert.assertEquals("write", h.getGrantInfo().getPrivilege());
- sawRole = true;
- }
- }
- Assert.assertTrue(sawUser && sawRole);
-
- hops = store.listPrincipalDBGrantsAll(roleNames[0], PrincipalType.ROLE);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listPrincipalDBGrantsAll(userNames[0], PrincipalType.USER);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.DATABASE, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listPrincipalDBGrantsAll(roleNames[1], PrincipalType.ROLE);
- Assert.assertEquals(0, hops.size());
- hops = store.listPrincipalDBGrantsAll(userNames[1], PrincipalType.USER);
- Assert.assertEquals(0, hops.size());
-
-
- } finally {
- store.dropDatabase(dbNames[0]);
- store.dropDatabase(dbNames[1]);
- }
- }
-
- @Test
- public void listGlobalGrants() throws Exception {
- String[] roleNames = new String[]{"lgg_role1", "lgg_role2"};
- String[] userNames = new String[]{"merry", "pippen"};
-
- store.addRole(roleNames[0], "me");
- store.addRole(roleNames[1], "me");
- int now = (int)(System.currentTimeMillis() / 1000);
-
- Role role1 = store.getRole(roleNames[0]);
- Role role2 = store.getRole(roleNames[1]);
- store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false);
- store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
- store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false);
-
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- HiveObjectRef hiveObjRef =
- new HiveObjectRef(HiveObjectType.GLOBAL, null, null, null, null);
- PrivilegeGrantInfo grantInfo =
- new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false);
- HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER,
- grantInfo);
- privileges.add(hop);
-
- grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
- hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
- privileges.add(hop);
-
- PrivilegeBag pBag = new PrivilegeBag(privileges);
- store.grantPrivileges(pBag);
-
- List<HiveObjectPrivilege> hops =
- store.listPrincipalGlobalGrants(roleNames[0], PrincipalType.ROLE);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.GLOBAL, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listPrincipalGlobalGrants(userNames[0], PrincipalType.USER);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.GLOBAL, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listPrincipalGlobalGrants(roleNames[1], PrincipalType.ROLE);
- Assert.assertEquals(0, hops.size());
- hops = store.listPrincipalGlobalGrants(userNames[1], PrincipalType.USER);
- Assert.assertEquals(0, hops.size());
-
- hops = store.listGlobalGrantsAll();
- Assert.assertEquals(2, hops.size());
- boolean sawUser = false, sawRole = false;
- for (HiveObjectPrivilege h : hops) {
- if (h.getPrincipalName().equals(userNames[0])) {
- Assert.assertEquals(PrincipalType.USER, h.getPrincipalType());
- Assert.assertEquals(HiveObjectType.GLOBAL, h.getHiveObject().getObjectType());
- Assert.assertEquals("read", h.getGrantInfo().getPrivilege());
- sawUser = true;
- } else if (h.getPrincipalName().equals(roleNames[0])) {
- Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType());
- Assert.assertEquals(HiveObjectType.GLOBAL, h.getHiveObject().getObjectType());
- Assert.assertEquals("write", h.getGrantInfo().getPrivilege());
- sawRole = true;
- }
- }
- Assert.assertTrue(sawUser && sawRole);
- }
-
- @Test
- public void listTableGrants() throws Exception {
- String dbName = "ltg_db";
- String[] tableNames = new String[] {"ltg_t1", "ltg_t2"};
- try {
- Database db = new Database(dbName, "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableNames[0], dbName, "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
- table = new Table(tableNames[1], dbName, "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
- String[] roleNames = new String[]{"ltg_role1", "ltg_role2"};
- String[] userNames = new String[]{"gandalf", "radagast"};
-
- store.addRole(roleNames[0], "me");
- store.addRole(roleNames[1], "me");
- int now = (int)(System.currentTimeMillis() / 1000);
-
- Role role1 = store.getRole(roleNames[0]);
- Role role2 = store.getRole(roleNames[1]);
- store.grantRole(role1, userNames[0], PrincipalType.USER, "bob", PrincipalType.USER, false);
- store.grantRole(role1, roleNames[1], PrincipalType.ROLE, "admin", PrincipalType.ROLE, true);
- store.grantRole(role2, userNames[1], PrincipalType.USER, "bob", PrincipalType.USER, false);
-
- List<HiveObjectPrivilege> privileges = new ArrayList<HiveObjectPrivilege>();
- HiveObjectRef hiveObjRef =
- new HiveObjectRef(HiveObjectType.TABLE, dbName, tableNames[0], null, null);
- PrivilegeGrantInfo grantInfo =
- new PrivilegeGrantInfo("read", now, "me", PrincipalType.USER, false);
- HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, userNames[0], PrincipalType.USER,
- grantInfo);
- privileges.add(hop);
-
- grantInfo = new PrivilegeGrantInfo("write", now, "me", PrincipalType.USER, true);
- hop = new HiveObjectPrivilege(hiveObjRef, roleNames[0], PrincipalType.ROLE, grantInfo);
- privileges.add(hop);
-
- PrivilegeBag pBag = new PrivilegeBag(privileges);
- store.grantPrivileges(pBag);
-
- List<HiveObjectPrivilege> hops =
- store.listAllTableGrants(roleNames[0], PrincipalType.ROLE, dbName, tableNames[0]);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listAllTableGrants(userNames[0], PrincipalType.USER, dbName, tableNames[0]);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listAllTableGrants(roleNames[1], PrincipalType.ROLE, dbName, tableNames[0]);
- Assert.assertEquals(0, hops.size());
- hops = store.listAllTableGrants(userNames[1], PrincipalType.USER, dbName, tableNames[0]);
- Assert.assertEquals(0, hops.size());
-
- hops = store.listAllTableGrants(roleNames[0], PrincipalType.ROLE, dbName, tableNames[1]);
- Assert.assertEquals(0, hops.size());
- hops = store.listAllTableGrants(userNames[0], PrincipalType.USER, dbName, tableNames[1]);
- Assert.assertEquals(0, hops.size());
-
- hops = store.listTableGrantsAll(dbName, tableNames[0]);
- Assert.assertEquals(2, hops.size());
- boolean sawUser = false, sawRole = false;
- for (HiveObjectPrivilege h : hops) {
- if (h.getPrincipalName().equals(userNames[0])) {
- Assert.assertEquals(PrincipalType.USER, h.getPrincipalType());
- Assert.assertEquals(HiveObjectType.TABLE, h.getHiveObject().getObjectType());
- Assert.assertEquals("read", h.getGrantInfo().getPrivilege());
- sawUser = true;
- } else if (h.getPrincipalName().equals(roleNames[0])) {
- Assert.assertEquals(PrincipalType.ROLE, h.getPrincipalType());
- Assert.assertEquals(HiveObjectType.TABLE, h.getHiveObject().getObjectType());
- Assert.assertEquals("write", h.getGrantInfo().getPrivilege());
- sawRole = true;
- }
- }
- Assert.assertTrue(sawUser && sawRole);
-
- hops = store.listPrincipalTableGrantsAll(roleNames[0], PrincipalType.ROLE);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.ROLE, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("write", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listPrincipalTableGrantsAll(userNames[0], PrincipalType.USER);
- Assert.assertEquals(1, hops.size());
- Assert.assertEquals(PrincipalType.USER, hops.get(0).getPrincipalType());
- Assert.assertEquals(HiveObjectType.TABLE, hops.get(0).getHiveObject().getObjectType());
- Assert.assertEquals("read", hops.get(0).getGrantInfo().getPrivilege());
-
- hops = store.listPrincipalDBGrantsAll(roleNames[1], PrincipalType.ROLE);
- Assert.assertEquals(0, hops.size());
- hops = store.listPrincipalDBGrantsAll(userNames[1], PrincipalType.USER);
- Assert.assertEquals(0, hops.size());
-
-
- } finally {
- store.dropTable(dbName, tableNames[0]);
- store.dropTable(dbName, tableNames[1]);
- store.dropDatabase(dbName);
- }
- }
-
- @Test
- public void tableStatistics() throws Exception {
- long now = System.currentTimeMillis();
- String dbname = "default";
- String tableName = "statstable";
- String boolcol = "boolcol";
- String longcol = "longcol";
- String doublecol = "doublecol";
- String stringcol = "stringcol";
- String binarycol = "bincol";
- String decimalcol = "deccol";
- long trues = 37;
- long falses = 12;
- long booleanNulls = 2;
- long longHigh = 120938479124L;
- long longLow = -12341243213412124L;
- long longNulls = 23;
- long longDVs = 213L;
- double doubleHigh = 123423.23423;
- double doubleLow = 0.00001234233;
- long doubleNulls = 92;
- long doubleDVs = 1234123421L;
- long strMaxLen = 1234;
- double strAvgLen = 32.3;
- long strNulls = 987;
- long strDVs = 906;
- long binMaxLen = 123412987L;
- double binAvgLen = 76.98;
- long binNulls = 976998797L;
- Decimal decHigh = new Decimal();
- decHigh.setScale((short)3);
- decHigh.setUnscaled("3876".getBytes()); // I have no clue how this is translated, but it
- // doesn't matter
- Decimal decLow = new Decimal();
- decLow.setScale((short)3);
- decLow.setUnscaled("38".getBytes());
- long decNulls = 13;
- long decDVs = 923947293L;
-
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema(boolcol, "boolean", "nocomment"));
- cols.add(new FieldSchema(longcol, "long", "nocomment"));
- cols.add(new FieldSchema(doublecol, "double", "nocomment"));
- cols.add(new FieldSchema(stringcol, "varchar(32)", "nocomment"));
- cols.add(new FieldSchema(binarycol, "binary", "nocomment"));
- cols.add(new FieldSchema(decimalcol, "decimal(5, 3)", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, dbname, "me", (int)now / 1000, (int)now / 1000, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- ColumnStatistics stats = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
- desc.setLastAnalyzed(now);
- desc.setDbName(dbname);
- desc.setTableName(tableName);
- desc.setIsTblLevel(true);
- stats.setStatsDesc(desc);
-
- // Do one column of each type
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName(boolcol);
- obj.setColType("boolean");
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData boolData = new BooleanColumnStatsData();
- boolData.setNumTrues(trues);
- boolData.setNumFalses(falses);
- boolData.setNumNulls(booleanNulls);
- data.setBooleanStats(boolData);
- obj.setStatsData(data);
- stats.addToStatsObj(obj);
-
- obj = new ColumnStatisticsObj();
- obj.setColName(longcol);
- obj.setColType("long");
- data = new ColumnStatisticsData();
- LongColumnStatsData longData = new LongColumnStatsData();
- longData.setHighValue(longHigh);
- longData.setLowValue(longLow);
- longData.setNumNulls(longNulls);
- longData.setNumDVs(longDVs);
- data.setLongStats(longData);
- obj.setStatsData(data);
- stats.addToStatsObj(obj);
-
- obj = new ColumnStatisticsObj();
- obj.setColName(doublecol);
- obj.setColType("double");
- data = new ColumnStatisticsData();
- DoubleColumnStatsData doubleData = new DoubleColumnStatsData();
- doubleData.setHighValue(doubleHigh);
- doubleData.setLowValue(doubleLow);
- doubleData.setNumNulls(doubleNulls);
- doubleData.setNumDVs(doubleDVs);
- data.setDoubleStats(doubleData);
- obj.setStatsData(data);
- stats.addToStatsObj(obj);
-
- store.updateTableColumnStatistics(stats);
-
- stats = store.getTableColumnStatistics(dbname, tableName,
- Arrays.asList(boolcol, longcol, doublecol));
-
- // We'll check all of the individual values later.
- Assert.assertEquals(3, stats.getStatsObjSize());
-
- // check that we can fetch just some of the columns
- stats = store.getTableColumnStatistics(dbname, tableName, Arrays.asList(boolcol));
- Assert.assertEquals(1, stats.getStatsObjSize());
-
- stats = new ColumnStatistics();
- stats.setStatsDesc(desc);
-
-
- obj = new ColumnStatisticsObj();
- obj.setColName(stringcol);
- obj.setColType("string");
- data = new ColumnStatisticsData();
- StringColumnStatsData strData = new StringColumnStatsData();
- strData.setMaxColLen(strMaxLen);
- strData.setAvgColLen(strAvgLen);
- strData.setNumNulls(strNulls);
- strData.setNumDVs(strDVs);
- data.setStringStats(strData);
- obj.setStatsData(data);
- stats.addToStatsObj(obj);
-
- obj = new ColumnStatisticsObj();
- obj.setColName(binarycol);
- obj.setColType("binary");
- data = new ColumnStatisticsData();
- BinaryColumnStatsData binData = new BinaryColumnStatsData();
- binData.setMaxColLen(binMaxLen);
- binData.setAvgColLen(binAvgLen);
- binData.setNumNulls(binNulls);
- data.setBinaryStats(binData);
- obj.setStatsData(data);
- stats.addToStatsObj(obj);
-
- obj = new ColumnStatisticsObj();
- obj.setColName(decimalcol);
- obj.setColType("decimal(5,3)");
- data = new ColumnStatisticsData();
- DecimalColumnStatsData decData = new DecimalColumnStatsData();
- LOG.debug("Setting decimal high value to " + decHigh.getScale() + " <" + new String(decHigh.getUnscaled()) + ">");
- decData.setHighValue(decHigh);
- decData.setLowValue(decLow);
- decData.setNumNulls(decNulls);
- decData.setNumDVs(decDVs);
- data.setDecimalStats(decData);
- obj.setStatsData(data);
- stats.addToStatsObj(obj);
-
- store.updateTableColumnStatistics(stats);
-
- stats = store.getTableColumnStatistics(dbname, tableName,
- Arrays.asList(boolcol, longcol, doublecol, stringcol, binarycol, decimalcol));
- Assert.assertEquals(now, stats.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(dbname, stats.getStatsDesc().getDbName());
- Assert.assertEquals(tableName, stats.getStatsDesc().getTableName());
- Assert.assertTrue(stats.getStatsDesc().isIsTblLevel());
-
- Assert.assertEquals(6, stats.getStatsObjSize());
-
- ColumnStatisticsData colData = stats.getStatsObj().get(0).getStatsData();
- Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS, colData.getSetField());
- boolData = colData.getBooleanStats();
- Assert.assertEquals(trues, boolData.getNumTrues());
- Assert.assertEquals(falses, boolData.getNumFalses());
- Assert.assertEquals(booleanNulls, boolData.getNumNulls());
-
- colData = stats.getStatsObj().get(1).getStatsData();
- Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, colData.getSetField());
- longData = colData.getLongStats();
- Assert.assertEquals(longHigh, longData.getHighValue());
- Assert.assertEquals(longLow, longData.getLowValue());
- Assert.assertEquals(longNulls, longData.getNumNulls());
- Assert.assertEquals(longDVs, longData.getNumDVs());
-
- colData = stats.getStatsObj().get(2).getStatsData();
- Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, colData.getSetField());
- doubleData = colData.getDoubleStats();
- Assert.assertEquals(doubleHigh, doubleData.getHighValue(), 0.01);
- Assert.assertEquals(doubleLow, doubleData.getLowValue(), 0.01);
- Assert.assertEquals(doubleNulls, doubleData.getNumNulls());
- Assert.assertEquals(doubleDVs, doubleData.getNumDVs());
-
- colData = stats.getStatsObj().get(3).getStatsData();
- Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, colData.getSetField());
- strData = colData.getStringStats();
- Assert.assertEquals(strMaxLen, strData.getMaxColLen());
- Assert.assertEquals(strAvgLen, strData.getAvgColLen(), 0.01);
- Assert.assertEquals(strNulls, strData.getNumNulls());
- Assert.assertEquals(strDVs, strData.getNumDVs());
-
- colData = stats.getStatsObj().get(4).getStatsData();
- Assert.assertEquals(ColumnStatisticsData._Fields.BINARY_STATS, colData.getSetField());
- binData = colData.getBinaryStats();
- Assert.assertEquals(binMaxLen, binData.getMaxColLen());
- Assert.assertEquals(binAvgLen, binData.getAvgColLen(), 0.01);
- Assert.assertEquals(binNulls, binData.getNumNulls());
-
- colData = stats.getStatsObj().get(5).getStatsData();
- Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, colData.getSetField());
- decData = colData.getDecimalStats();
- Assert.assertEquals(decHigh, decData.getHighValue());
- Assert.assertEquals(decLow, decData.getLowValue());
- Assert.assertEquals(decNulls, decData.getNumNulls());
- Assert.assertEquals(decDVs, decData.getNumDVs());
-
- }
-
- @Test
- public void partitionStatistics() throws Exception {
- long now = System.currentTimeMillis();
- String dbname = "default";
- String tableName = "statspart";
- String[] partNames = {"ds=today", "ds=yesterday"};
- String[] partVals = {"today", "yesterday"};
- String boolcol = "boolcol";
- String longcol = "longcol";
- String doublecol = "doublecol";
- String stringcol = "stringcol";
- String binarycol = "bincol";
- String decimalcol = "deccol";
- long trues = 37;
- long falses = 12;
- long booleanNulls = 2;
- long strMaxLen = 1234;
- double strAvgLen = 32.3;
- long strNulls = 987;
- long strDVs = 906;
-
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema(boolcol, "boolean", "nocomment"));
- cols.add(new FieldSchema(longcol, "long", "nocomment"));
- cols.add(new FieldSchema(doublecol, "double", "nocomment"));
- cols.add(new FieldSchema(stringcol, "varchar(32)", "nocomment"));
- cols.add(new FieldSchema(binarycol, "binary", "nocomment"));
- cols.add(new FieldSchema(decimalcol, "decimal(5, 3)", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbname, "me", (int)now / 1000, (int)now / 1000, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
- for (String partVal : partVals) {
- Partition part = new Partition(Arrays.asList(partVal), dbname, tableName, (int) now / 1000,
- (int) now / 1000, sd, emptyParameters);
- store.addPartition(part);
- }
-
- for (int i = 0; i < partNames.length; i++) {
- ColumnStatistics stats = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
- desc.setLastAnalyzed(now);
- desc.setDbName(dbname);
- desc.setTableName(tableName);
- desc.setIsTblLevel(false);
- desc.setPartName(partNames[i]);
- stats.setStatsDesc(desc);
-
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName(boolcol);
- obj.setColType("boolean");
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData boolData = new BooleanColumnStatsData();
- boolData.setNumTrues(trues);
- boolData.setNumFalses(falses);
- boolData.setNumNulls(booleanNulls);
- data.setBooleanStats(boolData);
- obj.setStatsData(data);
- stats.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i]));
- }
-
- List<ColumnStatistics> statsList = store.getPartitionColumnStatistics(dbname, tableName,
- Arrays.asList(partNames), Arrays.asList(boolcol));
-
- Assert.assertEquals(2, statsList.size());
- for (int i = 0; i < partNames.length; i++) {
- Assert.assertEquals(1, statsList.get(i).getStatsObjSize());
- }
-
- for (int i = 0; i < partNames.length; i++) {
- ColumnStatistics stats = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
- desc.setLastAnalyzed(now);
- desc.setDbName(dbname);
- desc.setTableName(tableName);
- desc.setIsTblLevel(false);
- desc.setPartName(partNames[i]);
- stats.setStatsDesc(desc);
-
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName(stringcol);
- obj.setColType("string");
- ColumnStatisticsData data = new ColumnStatisticsData();
- StringColumnStatsData strData = new StringColumnStatsData();
- strData.setMaxColLen(strMaxLen);
- strData.setAvgColLen(strAvgLen);
- strData.setNumNulls(strNulls);
- strData.setNumDVs(strDVs);
- data.setStringStats(strData);
- obj.setStatsData(data);
- stats.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(stats, Arrays.asList(partVals[i]));
- }
-
- // Make sure when we ask for one we only get one
- statsList = store.getPartitionColumnStatistics(dbname, tableName,
- Arrays.asList(partNames), Arrays.asList(boolcol));
-
- Assert.assertEquals(2, statsList.size());
- for (int i = 0; i < partNames.length; i++) {
- Assert.assertEquals(1, statsList.get(i).getStatsObjSize());
- }
-
- statsList = store.getPartitionColumnStatistics(dbname, tableName,
- Arrays.asList(partNames), Arrays.asList(boolcol, stringcol));
-
- Assert.assertEquals(2, statsList.size());
- for (int i = 0; i < partNames.length; i++) {
- Assert.assertEquals(2, statsList.get(i).getStatsObjSize());
- // Just check one piece of the data, I don't need to check it all again
- Assert.assertEquals(booleanNulls,
- statsList.get(i).getStatsObj().get(0).getStatsData().getBooleanStats().getNumNulls());
- Assert.assertEquals(strDVs,
- statsList.get(i).getStatsObj().get(1).getStatsData().getStringStats().getNumDVs());
- }
- }
-
- @Test
- public void delegationToken() throws Exception {
- store.addToken("abc", "def");
- store.addToken("ghi", "jkl");
-
- Assert.assertEquals("def", store.getToken("abc"));
- Assert.assertEquals("jkl", store.getToken("ghi"));
- Assert.assertNull(store.getToken("wabawaba"));
- String[] allToks = store.getAllTokenIdentifiers().toArray(new String[2]);
- Arrays.sort(allToks);
- Assert.assertArrayEquals(new String[]{"abc", "ghi"}, allToks);
-
- store.removeToken("abc");
- store.removeToken("wabawaba");
-
- Assert.assertNull(store.getToken("abc"));
- Assert.assertEquals("jkl", store.getToken("ghi"));
- allToks = store.getAllTokenIdentifiers().toArray(new String[1]);
- Assert.assertArrayEquals(new String[]{"ghi"}, allToks);
- }
-
- @Test
- public void masterKey() throws Exception {
- Assert.assertEquals(0, store.addMasterKey("k1"));
- Assert.assertEquals(1, store.addMasterKey("k2"));
-
- String[] keys = store.getMasterKeys();
- Arrays.sort(keys);
- Assert.assertArrayEquals(new String[]{"k1", "k2"}, keys);
-
- store.updateMasterKey(0, "k3");
- keys = store.getMasterKeys();
- Arrays.sort(keys);
- Assert.assertArrayEquals(new String[]{"k2", "k3"}, keys);
-
- store.removeMasterKey(1);
- keys = store.getMasterKeys();
- Assert.assertArrayEquals(new String[]{"k3"}, keys);
-
- thrown.expect(NoSuchObjectException.class);
- store.updateMasterKey(72, "whatever");
- }
-
-}
[11/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
new file mode 100644
index 0000000..2542a00
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.Date;
+import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
+
+public class DateColumnStatsMerger extends ColumnStatsMerger {
+ @Override
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ DateColumnStatsData aggregateData = aggregateColStats.getStatsData().getDateStats();
+ DateColumnStatsData newData = newColStats.getStatsData().getDateStats();
+ Date lowValue = aggregateData.getLowValue().compareTo(newData.getLowValue()) < 0 ? aggregateData
+ .getLowValue() : newData.getLowValue();
+ aggregateData.setLowValue(lowValue);
+ Date highValue = aggregateData.getHighValue().compareTo(newData.getHighValue()) >= 0 ? aggregateData
+ .getHighValue() : newData.getHighValue();
+ aggregateData.setHighValue(highValue);
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+ || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ } else {
+ NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+ NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors());
+ long ndv = -1;
+ if (oldEst.canMerge(newEst)) {
+ oldEst.mergeEstimators(newEst);
+ ndv = oldEst.estimateNumDistinctValues();
+ aggregateData.setBitVectors(oldEst.serialize());
+ } else {
+ ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+ }
+ LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+ + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+ aggregateData.setNumDVs(ndv);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
new file mode 100644
index 0000000..4e8e129
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.Decimal;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+
+public class DecimalColumnStatsMerger extends ColumnStatsMerger {
+ @Override
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ DecimalColumnStatsData aggregateData = aggregateColStats.getStatsData().getDecimalStats();
+ DecimalColumnStatsData newData = newColStats.getStatsData().getDecimalStats();
+ Decimal lowValue = aggregateData.getLowValue() != null
+ && (aggregateData.getLowValue().compareTo(newData.getLowValue()) > 0) ? aggregateData
+ .getLowValue() : newData.getLowValue();
+ aggregateData.setLowValue(lowValue);
+ Decimal highValue = aggregateData.getHighValue() != null
+ && (aggregateData.getHighValue().compareTo(newData.getHighValue()) > 0) ? aggregateData
+ .getHighValue() : newData.getHighValue();
+ aggregateData.setHighValue(highValue);
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+ || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ } else {
+ NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+ NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors());
+ long ndv = -1;
+ if (oldEst.canMerge(newEst)) {
+ oldEst.mergeEstimators(newEst);
+ ndv = oldEst.estimateNumDistinctValues();
+ aggregateData.setBitVectors(oldEst.serialize());
+ } else {
+ ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+ }
+ LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+ + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+ aggregateData.setNumDVs(ndv);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
new file mode 100644
index 0000000..4ef5c39
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+
+public class DoubleColumnStatsMerger extends ColumnStatsMerger {
+ @Override
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ DoubleColumnStatsData aggregateData = aggregateColStats.getStatsData().getDoubleStats();
+ DoubleColumnStatsData newData = newColStats.getStatsData().getDoubleStats();
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+ || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ } else {
+ NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+ NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors());
+ long ndv = -1;
+ if (oldEst.canMerge(newEst)) {
+ oldEst.mergeEstimators(newEst);
+ ndv = oldEst.estimateNumDistinctValues();
+ aggregateData.setBitVectors(oldEst.serialize());
+ } else {
+ ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+ }
+ LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+ + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+ aggregateData.setNumDVs(ndv);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
new file mode 100644
index 0000000..acf7f03
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+
+public class LongColumnStatsMerger extends ColumnStatsMerger {
+ @Override
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ LongColumnStatsData aggregateData = aggregateColStats.getStatsData().getLongStats();
+ LongColumnStatsData newData = newColStats.getStatsData().getLongStats();
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+ || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ } else {
+ NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+ NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors());
+ long ndv = -1;
+ if (oldEst.canMerge(newEst)) {
+ oldEst.mergeEstimators(newEst);
+ ndv = oldEst.estimateNumDistinctValues();
+ aggregateData.setBitVectors(oldEst.serialize());
+ } else {
+ ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+ }
+ LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+ + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+ aggregateData.setNumDVs(ndv);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
new file mode 100644
index 0000000..b3cd33c
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+
+public class StringColumnStatsMerger extends ColumnStatsMerger {
+ @Override
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ StringColumnStatsData aggregateData = aggregateColStats.getStatsData().getStringStats();
+ StringColumnStatsData newData = newColStats.getStatsData().getStringStats();
+ aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
+ aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
+ || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ } else {
+ NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(aggregateData.getBitVectors());
+ NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors());
+ long ndv = -1;
+ if (oldEst.canMerge(newEst)) {
+ oldEst.mergeEstimators(newEst);
+ ndv = oldEst.estimateNumDistinctValues();
+ aggregateData.setBitVectors(oldEst.serialize());
+ } else {
+ ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
+ }
+ LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+ + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
+ aggregateData.setNumDVs(ndv);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/AggrStatsInvalidatorFilter.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/AggrStatsInvalidatorFilter.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/AggrStatsInvalidatorFilter.java
deleted file mode 100644
index 2db5c38..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/AggrStatsInvalidatorFilter.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.primitives.Longs;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hive.common.util.BloomFilter;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Filter for scanning aggregates stats table
- */
-public class AggrStatsInvalidatorFilter extends FilterBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(AggrStatsInvalidatorFilter.class.getName());
- private final List<HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry> entries;
- private final long runEvery;
- private final long maxCacheEntryLife;
- // This class is not serializable, so I realize transient doesn't mean anything. It's just to
- // comunicate that we don't serialize this and ship it across to the filter on the other end.
- // We use the time the filter is actually instantiated in HBase.
- private transient long now;
-
- public static Filter parseFrom(byte[] serialized) throws DeserializationException {
- try {
- return new AggrStatsInvalidatorFilter(
- HbaseMetastoreProto.AggrStatsInvalidatorFilter.parseFrom(serialized));
- } catch (InvalidProtocolBufferException e) {
- throw new DeserializationException(e);
- }
- }
-
- /**
- * @param proto Protocol buffer representation of this filter.
- */
- AggrStatsInvalidatorFilter(HbaseMetastoreProto.AggrStatsInvalidatorFilter proto) {
- this.entries = proto.getToInvalidateList();
- this.runEvery = proto.getRunEvery();
- this.maxCacheEntryLife = proto.getMaxCacheEntryLife();
- now = System.currentTimeMillis();
- }
-
- @Override
- public byte[] toByteArray() throws IOException {
- return HbaseMetastoreProto.AggrStatsInvalidatorFilter.newBuilder()
- .addAllToInvalidate(entries)
- .setRunEvery(runEvery)
- .setMaxCacheEntryLife(maxCacheEntryLife)
- .build()
- .toByteArray();
- }
-
- @Override
- public boolean filterAllRemaining() throws IOException {
- return false;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell cell) throws IOException {
- // Is this the partition we want?
- if (Arrays.equals(CellUtil.cloneQualifier(cell), HBaseReadWrite.AGGR_STATS_BLOOM_COL)) {
- HbaseMetastoreProto.AggrStatsBloomFilter fromCol =
- HbaseMetastoreProto.AggrStatsBloomFilter.parseFrom(CellUtil.cloneValue(cell));
- BloomFilter bloom = null;
- if (now - maxCacheEntryLife > fromCol.getAggregatedAt()) {
- // It's too old, kill it regardless of whether we were asked to or not.
- return ReturnCode.INCLUDE;
- } else if (now - runEvery * 2 <= fromCol.getAggregatedAt()) {
- // It's too new. We might be stomping on something that was just created. Skip it.
- return ReturnCode.NEXT_ROW;
- } else {
- // Look through each of our entries and see if any of them match.
- for (HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry entry : entries) {
- // First check if we match on db and table match
- if (entry.getDbName().equals(fromCol.getDbName()) &&
- entry.getTableName().equals(fromCol.getTableName())) {
- if (bloom == null) {
- // Now, reconstitute the bloom filter and probe it with each of our partition names
- List<Long> bitsList = fromCol.getBloomFilter().getBitsList();
- bloom = new BloomFilter(Longs.toArray(bitsList), fromCol.getBloomFilter().getNumFuncs());
- }
- if (bloom.test(entry.getPartName().toByteArray())) {
- // This is most likely a match, so mark it and quit looking.
- return ReturnCode.INCLUDE;
- }
- }
- }
- }
- return ReturnCode.NEXT_ROW;
- } else {
- return ReturnCode.NEXT_COL;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/Counter.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/Counter.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/Counter.java
deleted file mode 100644
index 2359939..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/Counter.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * A simple metric to count how many times something occurs.
- */
-class Counter {
- private final String name;
- private long cnt;
-
- Counter(String name) {
- this.name = name;
- cnt = 0;
- }
-
- void incr() {
- cnt++;
- }
-
- void clear() {
- cnt = 0;
- }
-
- String dump() {
- StringBuilder bldr = new StringBuilder("Dumping metric: ");
- bldr.append(name).append(' ').append(cnt);
- return bldr.toString();
- }
-
- @VisibleForTesting long getCnt() {
- return cnt;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseConnection.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseConnection.java
deleted file mode 100644
index 696e588..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseConnection.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * A connection to HBase. Separated out as an interface so we can slide different transaction
- * managers between our code and HBase.
- */
-public interface HBaseConnection extends Configurable {
-
- /**
- * Connects to HBase. This must be called after {@link #setConf} has been called.
- * @throws IOException
- */
- void connect() throws IOException;
-
- /**
- * Close the connection. No further operations are possible after this is done.
- * @throws IOException
- */
- void close() throws IOException;
-
- /**
- * Begin a transaction.
- * @throws IOException
- */
- void beginTransaction() throws IOException;
-
- /**
- * Commit a transaction
- * @throws IOException indicates the commit has failed
- */
- void commitTransaction() throws IOException;
-
- /**
- * Rollback a transaction
- * @throws IOException
- */
- void rollbackTransaction() throws IOException;
-
- /**
- * Flush commits. A no-op for transaction implementations since they will write at commit time.
- * @param htab Table to flush
- * @throws IOException
- */
- void flush(HTableInterface htab) throws IOException;
-
- /**
- * Create a new table
- * @param tableName name of the table
- * @param columnFamilies name of the column families in the table
- * @throws IOException
- */
- void createHBaseTable(String tableName, List<byte[]> columnFamilies) throws IOException;
-
- /**
- * Fetch an existing HBase table.
- * @param tableName name of the table
- * @return table handle
- * @throws IOException
- */
- HTableInterface getHBaseTable(String tableName) throws IOException;
-
- /**
- * Fetch an existing HBase table and force a connection to it. This should be used only in
- * cases where you want to assure that the table exists (ie at install).
- * @param tableName name of the table
- * @param force if true, force a connection by fetching a non-existant key
- * @return table handle
- * @throws IOException
- */
- HTableInterface getHBaseTable(String tableName, boolean force) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseFilterPlanUtil.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseFilterPlanUtil.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseFilterPlanUtil.java
deleted file mode 100644
index 3c03846..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseFilterPlanUtil.java
+++ /dev/null
@@ -1,612 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.hbase.PartitionKeyComparator.Operator;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-
-
-/**
- * Utility function for generating hbase partition filtering plan representation
- * from ExpressionTree.
- * Optimizations to be done -
- * - Case where all partition keys are specified. Should use a get
- *
- * {@link PartitionFilterGenerator} is a visitor on the given filter expression tree. After
- * walking it it produces the HBase execution plan represented by {@link FilterPlan}. See
- * their javadocs for more details.
- */
-class HBaseFilterPlanUtil {
-
- /**
- * Compare two byte arrays.
- *
- * @param ar1
- * first byte array
- * @param ar2
- * second byte array
- * @return -1 if ar1 < ar2, 0 if == , 1 if >
- */
- static int compare(byte[] ar1, byte[] ar2) {
- // null check is not needed, nulls are not passed here
- for (int i = 0; i < ar1.length; i++) {
- if (i == ar2.length) {
- return 1;
- } else {
- if (ar1[i] == ar2[i]) {
- continue;
- } else if (ar1[i] > ar2[i]) {
- return 1;
- } else {
- return -1;
- }
- }
- }
- // ar2 equal until length of ar1.
- if(ar1.length == ar2.length) {
- return 0;
- }
- // ar2 has more bytes
- return -1;
- }
-
- /**
- * Represents the execution plan for hbase to find the set of partitions that
- * match given filter expression.
- * If you have an AND or OR of two expressions, you can determine FilterPlan for each
- * children and then call lhs.and(rhs) or lhs.or(rhs) respectively
- * to generate a new plan for the expression.
- *
- * The execution plan has one or more ScanPlan objects. To get the results the set union of all
- * ScanPlan objects needs to be done.
- */
- public static abstract class FilterPlan {
- abstract FilterPlan and(FilterPlan other);
- abstract FilterPlan or(FilterPlan other);
- abstract List<ScanPlan> getPlans();
- @Override
- public String toString() {
- return getPlans().toString();
- }
-
- }
-
- /**
- * Represents a union/OR of single scan plans (ScanPlan).
- */
- public static class MultiScanPlan extends FilterPlan {
- final ImmutableList<ScanPlan> scanPlans;
-
- public MultiScanPlan(List<ScanPlan> scanPlans){
- this.scanPlans = ImmutableList.copyOf(scanPlans);
- }
-
- @Override
- public FilterPlan and(FilterPlan other) {
- // Convert to disjunctive normal form (DNF), ie OR of ANDs
- // First get a new set of FilterPlans by doing an AND
- // on each ScanPlan in this one with the other FilterPlan
- List<FilterPlan> newFPlans = new ArrayList<FilterPlan>();
- for (ScanPlan splan : getPlans()) {
- newFPlans.add(splan.and(other));
- }
- //now combine scanPlans in multiple new FilterPlans into one
- // MultiScanPlan
- List<ScanPlan> newScanPlans = new ArrayList<ScanPlan>();
- for (FilterPlan fp : newFPlans) {
- newScanPlans.addAll(fp.getPlans());
- }
- return new MultiScanPlan(newScanPlans);
- }
-
- @Override
- public FilterPlan or(FilterPlan other) {
- // just combine the ScanPlans
- List<ScanPlan> newScanPlans = new ArrayList<ScanPlan>(this.getPlans());
- newScanPlans.addAll(other.getPlans());
- return new MultiScanPlan(newScanPlans);
- }
-
- @Override
- public List<ScanPlan> getPlans() {
- return scanPlans;
- }
- }
-
- /**
- * Represents a single Hbase Scan api call
- */
- public static class ScanPlan extends FilterPlan {
-
- public static class ScanMarker {
- final String value;
- /**
- * If inclusive = true, it means that the
- * marker includes those bytes.
- * If it is false, it means the marker starts at the next possible byte array
- * or ends at the next possible byte array
- */
- final boolean isInclusive;
- final String type;
- ScanMarker(String obj, boolean i, String type){
- this.value = obj;
- this.isInclusive = i;
- this.type = type;
- }
- @Override
- public String toString() {
- return "ScanMarker [" + "value=" + value.toString() + ", isInclusive=" + isInclusive +
- ", type=" + type + "]";
- }
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + value.hashCode();
- result = prime * result + (isInclusive ? 1231 : 1237);
- result = prime * result + type.hashCode();
- return result;
- }
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- ScanMarker other = (ScanMarker) obj;
- if (!value.equals(other.value))
- return false;
- if (isInclusive != other.isInclusive)
- return false;
- if (type != other.type)
- return false;
- return true;
- }
- }
- public static class ScanMarkerPair {
- public ScanMarkerPair(ScanMarker startMarker, ScanMarker endMarker) {
- this.startMarker = startMarker;
- this.endMarker = endMarker;
- }
- ScanMarker startMarker;
- ScanMarker endMarker;
- }
- // represent Scan start, partition key name -> scanMarkerPair
- Map<String, ScanMarkerPair> markers = new HashMap<String, ScanMarkerPair>();
- List<Operator> ops = new ArrayList<Operator>();
-
- // Get the number of partition key prefixes which can be used in the scan range.
- // For example, if partition key is (year, month, state)
- // 1. year = 2015 and month >= 1 and month < 5
- // year + month can be used in scan range, majorParts = 2
- // 2. year = 2015 and state = 'CA'
- // only year can be used in scan range, majorParts = 1
- // 3. month = 10 and state = 'CA'
- // nothing can be used in scan range, majorParts = 0
- private int getMajorPartsCount(List<FieldSchema> parts) {
- int majorPartsCount = 0;
- while (majorPartsCount<parts.size() && markers.containsKey(parts.get(majorPartsCount).getName())) {
- ScanMarkerPair pair = markers.get(parts.get(majorPartsCount).getName());
- majorPartsCount++;
- if (pair.startMarker!=null && pair.endMarker!=null && pair.startMarker.value.equals(pair
- .endMarker.value) && pair.startMarker.isInclusive && pair.endMarker.isInclusive) {
- // is equal
- continue;
- } else {
- break;
- }
- }
- return majorPartsCount;
- }
- public Filter getFilter(List<FieldSchema> parts) {
- int majorPartsCount = getMajorPartsCount(parts);
- Set<String> majorKeys = new HashSet<String>();
- for (int i=0;i<majorPartsCount;i++) {
- majorKeys.add(parts.get(i).getName());
- }
-
- List<String> names = HBaseUtils.getPartitionNames(parts);
- List<PartitionKeyComparator.Range> ranges = new ArrayList<PartitionKeyComparator.Range>();
- for (Map.Entry<String, ScanMarkerPair> entry : markers.entrySet()) {
- if (names.contains(entry.getKey()) && !majorKeys.contains(entry.getKey())) {
- PartitionKeyComparator.Mark startMark = null;
- if (entry.getValue().startMarker != null) {
- startMark = new PartitionKeyComparator.Mark(entry.getValue().startMarker.value,
- entry.getValue().startMarker.isInclusive);
- }
- PartitionKeyComparator.Mark endMark = null;
- if (entry.getValue().endMarker != null) {
- startMark = new PartitionKeyComparator.Mark(entry.getValue().endMarker.value,
- entry.getValue().endMarker.isInclusive);
- }
- PartitionKeyComparator.Range range = new PartitionKeyComparator.Range(
- entry.getKey(), startMark, endMark);
- ranges.add(range);
- }
- }
-
- if (ranges.isEmpty() && ops.isEmpty()) {
- return null;
- } else {
- return new RowFilter(CompareFilter.CompareOp.EQUAL, new PartitionKeyComparator(
- StringUtils.join(names, ","), StringUtils.join(HBaseUtils.getPartitionKeyTypes(parts), ","),
- ranges, ops));
- }
- }
-
- public void setStartMarker(String keyName, String keyType, String start, boolean isInclusive) {
- if (markers.containsKey(keyName)) {
- markers.get(keyName).startMarker = new ScanMarker(start, isInclusive, keyType);
- } else {
- ScanMarkerPair marker = new ScanMarkerPair(new ScanMarker(start, isInclusive, keyType), null);
- markers.put(keyName, marker);
- }
- }
-
- public ScanMarker getStartMarker(String keyName) {
- if (markers.containsKey(keyName)) {
- return markers.get(keyName).startMarker;
- } else {
- return null;
- }
- }
-
- public void setEndMarker(String keyName, String keyType, String end, boolean isInclusive) {
- if (markers.containsKey(keyName)) {
- markers.get(keyName).endMarker = new ScanMarker(end, isInclusive, keyType);
- } else {
- ScanMarkerPair marker = new ScanMarkerPair(null, new ScanMarker(end, isInclusive, keyType));
- markers.put(keyName, marker);
- }
- }
-
- public ScanMarker getEndMarker(String keyName) {
- if (markers.containsKey(keyName)) {
- return markers.get(keyName).endMarker;
- } else {
- return null;
- }
- }
-
- @Override
- public FilterPlan and(FilterPlan other) {
- List<ScanPlan> newSPlans = new ArrayList<ScanPlan>();
- for (ScanPlan otherSPlan : other.getPlans()) {
- newSPlans.add(this.and(otherSPlan));
- }
- return new MultiScanPlan(newSPlans);
- }
-
- private ScanPlan and(ScanPlan other) {
- // create combined FilterPlan based on existing lhs and rhs plan
- ScanPlan newPlan = new ScanPlan();
- newPlan.markers.putAll(markers);
-
- for (String keyName : other.markers.keySet()) {
- if (newPlan.markers.containsKey(keyName)) {
- // create new scan start
- ScanMarker greaterStartMarker = getComparedMarker(this.getStartMarker(keyName),
- other.getStartMarker(keyName), true);
- if (greaterStartMarker != null) {
- newPlan.setStartMarker(keyName, greaterStartMarker.type, greaterStartMarker.value, greaterStartMarker.isInclusive);
- }
-
- // create new scan end
- ScanMarker lesserEndMarker = getComparedMarker(this.getEndMarker(keyName), other.getEndMarker(keyName),
- false);
- if (lesserEndMarker != null) {
- newPlan.setEndMarker(keyName, lesserEndMarker.type, lesserEndMarker.value, lesserEndMarker.isInclusive);
- }
- } else {
- newPlan.markers.put(keyName, other.markers.get(keyName));
- }
- }
-
- newPlan.ops.addAll(ops);
- newPlan.ops.addAll(other.ops);
- return newPlan;
- }
-
- /**
- * @param lStartMarker
- * @param rStartMarker
- * @param getGreater if true return greater startmarker, else return smaller one
- * @return greater/lesser marker depending on value of getGreater
- */
- @VisibleForTesting
- static ScanMarker getComparedMarker(ScanMarker lStartMarker, ScanMarker rStartMarker,
- boolean getGreater) {
- // if one of them has null bytes, just return other
- if(lStartMarker == null) {
- return rStartMarker;
- } else if (rStartMarker == null) {
- return lStartMarker;
- }
- TypeInfo expectedType =
- TypeInfoUtils.getTypeInfoFromTypeString(lStartMarker.type);
- ObjectInspector outputOI =
- TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
- Converter lConverter = ObjectInspectorConverters.getConverter(
- PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
- Converter rConverter = ObjectInspectorConverters.getConverter(
- PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
- Comparable lValue = (Comparable)lConverter.convert(lStartMarker.value);
- Comparable rValue = (Comparable)rConverter.convert(rStartMarker.value);
-
- int compareRes = lValue.compareTo(rValue);
- if (compareRes == 0) {
- // bytes are equal, now compare the isInclusive flags
- if (lStartMarker.isInclusive == rStartMarker.isInclusive) {
- // actually equal, so return any one
- return lStartMarker;
- }
- boolean isInclusive = true;
- // one that does not include the current bytes is greater
- if (getGreater) {
- isInclusive = false;
- }
- // else
- return new ScanMarker(lStartMarker.value, isInclusive, lStartMarker.type);
- }
- if (getGreater) {
- return compareRes == 1 ? lStartMarker : rStartMarker;
- }
- // else
- return compareRes == -1 ? lStartMarker : rStartMarker;
- }
-
-
- @Override
- public FilterPlan or(FilterPlan other) {
- List<ScanPlan> plans = new ArrayList<ScanPlan>(getPlans());
- plans.addAll(other.getPlans());
- return new MultiScanPlan(plans);
- }
-
- @Override
- public List<ScanPlan> getPlans() {
- return Arrays.asList(this);
- }
-
-
- /**
- * @return row suffix - This is appended to db + table, to generate start row for the Scan
- */
- public byte[] getStartRowSuffix(String dbName, String tableName, List<FieldSchema> parts) {
- int majorPartsCount = getMajorPartsCount(parts);
- List<String> majorPartTypes = new ArrayList<String>();
- List<String> components = new ArrayList<String>();
- boolean endPrefix = false;
- for (int i=0;i<majorPartsCount;i++) {
- majorPartTypes.add(parts.get(i).getType());
- ScanMarker marker = markers.get(parts.get(i).getName()).startMarker;
- if (marker != null) {
- components.add(marker.value);
- if (i==majorPartsCount-1) {
- endPrefix = !marker.isInclusive;
- }
- } else {
- components.add(null);
- if (i==majorPartsCount-1) {
- endPrefix = false;
- }
- }
- }
- byte[] bytes = HBaseUtils.buildPartitionKey(dbName, tableName, majorPartTypes, components, endPrefix);
- return bytes;
- }
-
- /**
- * @return row suffix - This is appended to db + table, to generate end row for the Scan
- */
- public byte[] getEndRowSuffix(String dbName, String tableName, List<FieldSchema> parts) {
- int majorPartsCount = getMajorPartsCount(parts);
- List<String> majorPartTypes = new ArrayList<String>();
- List<String> components = new ArrayList<String>();
- boolean endPrefix = false;
- for (int i=0;i<majorPartsCount;i++) {
- majorPartTypes.add(parts.get(i).getType());
- ScanMarker marker = markers.get(parts.get(i).getName()).endMarker;
- if (marker != null) {
- components.add(marker.value);
- if (i==majorPartsCount-1) {
- endPrefix = marker.isInclusive;
- }
- } else {
- components.add(null);
- if (i==majorPartsCount-1) {
- endPrefix = true;
- }
- }
- }
- byte[] bytes = HBaseUtils.buildPartitionKey(dbName, tableName, majorPartTypes, components, endPrefix);
- if (components.isEmpty()) {
- bytes[bytes.length-1]++;
- }
- return bytes;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("ScanPlan:\n");
- for (Map.Entry<String, ScanMarkerPair> entry : markers.entrySet()) {
- sb.append("key=" + entry.getKey() + "[startMarker=" + entry.getValue().startMarker
- + ", endMarker=" + entry.getValue().endMarker + "]");
- }
- return sb.toString();
- }
-
- }
-
- /**
- * Visitor for ExpressionTree.
- * It first generates the ScanPlan for the leaf nodes. The higher level nodes are
- * either AND or OR operations. It then calls FilterPlan.and and FilterPlan.or with
- * the child nodes to generate the plans for higher level nodes.
- */
- @VisibleForTesting
- static class PartitionFilterGenerator extends TreeVisitor {
- private FilterPlan curPlan;
-
- // this tells us if there is a condition that did not get included in the plan
- // such condition would be treated as getting evaluated to TRUE
- private boolean hasUnsupportedCondition = false;
-
- //Need to cache the left plans for the TreeNode. Use IdentityHashMap here
- // as we don't want to dedupe on two TreeNode that are otherwise considered equal
- Map<TreeNode, FilterPlan> leftPlans = new IdentityHashMap<TreeNode, FilterPlan>();
-
- // temporary params for current left and right side plans, for AND, OR
- private FilterPlan rPlan;
-
- private Map<String, String> nameToType = new HashMap<String, String>();
-
- public PartitionFilterGenerator(List<FieldSchema> parts) {
- for (FieldSchema part : parts) {
- nameToType.put(part.getName(), part.getType());
- }
- }
-
- FilterPlan getPlan() {
- return curPlan;
- }
-
- @Override
- protected void beginTreeNode(TreeNode node) throws MetaException {
- // reset the params
- curPlan = rPlan = null;
- }
-
- @Override
- protected void midTreeNode(TreeNode node) throws MetaException {
- leftPlans.put(node, curPlan);
- curPlan = null;
- }
-
- @Override
- protected void endTreeNode(TreeNode node) throws MetaException {
- rPlan = curPlan;
- FilterPlan lPlan = leftPlans.get(node);
- leftPlans.remove(node);
-
- switch (node.getAndOr()) {
- case AND:
- curPlan = lPlan.and(rPlan);
- break;
- case OR:
- curPlan = lPlan.or(rPlan);
- break;
- default:
- throw new AssertionError("Unexpected logical operation " + node.getAndOr());
- }
-
- }
-
-
- @Override
- public void visit(LeafNode node) throws MetaException {
- ScanPlan leafPlan = new ScanPlan();
- curPlan = leafPlan;
-
- // this is a condition on first partition column, so might influence the
- // start and end of the scan
- final boolean INCLUSIVE = true;
- switch (node.operator) {
- case EQUALS:
- leafPlan.setStartMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), INCLUSIVE);
- leafPlan.setEndMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), INCLUSIVE);
- break;
- case GREATERTHAN:
- leafPlan.setStartMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), !INCLUSIVE);
- break;
- case GREATERTHANOREQUALTO:
- leafPlan.setStartMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), INCLUSIVE);
- break;
- case LESSTHAN:
- leafPlan.setEndMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), !INCLUSIVE);
- break;
- case LESSTHANOREQUALTO:
- leafPlan.setEndMarker(node.keyName, nameToType.get(node.keyName), node.value.toString(), INCLUSIVE);
- break;
- case LIKE:
- leafPlan.ops.add(new Operator(Operator.Type.LIKE, node.keyName, node.value.toString()));
- break;
- case NOTEQUALS:
- case NOTEQUALS2:
- leafPlan.ops.add(new Operator(Operator.Type.NOTEQUALS, node.keyName, node.value.toString()));
- break;
- }
- }
-
- private boolean hasUnsupportedCondition() {
- return hasUnsupportedCondition;
- }
-
- }
-
- public static class PlanResult {
- public final FilterPlan plan;
- public final boolean hasUnsupportedCondition;
- PlanResult(FilterPlan plan, boolean hasUnsupportedCondition) {
- this.plan = plan;
- this.hasUnsupportedCondition = hasUnsupportedCondition;
- }
- }
-
- public static PlanResult getFilterPlan(ExpressionTree exprTree, List<FieldSchema> parts) throws MetaException {
- if (exprTree == null) {
- // TODO: if exprTree is null, we should do what ObjectStore does. See HIVE-10102
- return new PlanResult(new ScanPlan(), true);
- }
- PartitionFilterGenerator pGenerator = new PartitionFilterGenerator(parts);
- exprTree.accept(pGenerator);
- return new PlanResult(pGenerator.getPlan(), pGenerator.hasUnsupportedCondition());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseImport.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseImport.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseImport.java
deleted file mode 100644
index 5f89769..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseImport.java
+++ /dev/null
@@ -1,619 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.Deadline;
-import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A tool to take the contents of an RDBMS based Hive metastore and import it into an HBase based
- * one. To use this the config files for Hive configured to work with the RDBMS (that is,
- * including the JDBC string, etc.) as well as HBase configuration files must be in the path.
- * There should not be a hive-site.xml that specifies HBaseStore in the path. This tool will then
- * handle connecting to the RDBMS via the {@link org.apache.hadoop.hive.metastore.ObjectStore}
- * and HBase via {@link org.apache.hadoop.hive.metastore.hbase.HBaseStore} and transferring the
- * data.
- *
- * This tool can import an entire metastore or only selected objects. When selecting objects it
- * is necessary to fully specify the object's name. For example, if you want to import the table
- * T in the default database it needs to be identified as default.T. The same is true for
- * functions. When an object is specified, everything under that object will be imported (e.g.
- * if you select database D, then all tables and functions in that database will be
- * imported as well).
- *
- * At this point only tables and partitions are handled in parallel as it is assumed there are
- * relatively few of everything else.
- *
- * Note that HBaseSchemaTool must have already been used to create the appropriate tables in HBase.
- */
-public class HBaseImport {
-
- static final private Logger LOG = LoggerFactory.getLogger(HBaseImport.class.getName());
-
- public static int main(String[] args) {
- try {
- HBaseImport tool = new HBaseImport();
- int rv = tool.init(args);
- if (rv != 0) return rv;
- tool.run();
- } catch (Exception e) {
- System.err.println("Caught exception " + e.getClass().getName() + " with message <" +
- e.getMessage() + ">");
- return 1;
- }
- return 0;
- }
-
- private ThreadLocal<RawStore> rdbmsStore = new ThreadLocal<RawStore>() {
- @Override
- protected RawStore initialValue() {
- if (rdbmsConf == null) {
- throw new RuntimeException("order violation, need to set rdbms conf first");
- }
- RawStore os = new ObjectStore();
- os.setConf(rdbmsConf);
- return os;
- }
- };
-
- private ThreadLocal<RawStore> hbaseStore = new ThreadLocal<RawStore>() {
- @Override
- protected RawStore initialValue() {
- if (hbaseConf == null) {
- throw new RuntimeException("order violation, need to set hbase conf first");
- }
- RawStore hs = new HBaseStore();
- hs.setConf(hbaseConf);
- return hs;
- }
- };
-
- private Configuration rdbmsConf;
- private Configuration hbaseConf;
- private List<Database> dbs;
- private BlockingQueue<Table> partitionedTables;
- private BlockingQueue<String[]> tableNameQueue;
- private BlockingQueue<String[]> indexNameQueue;
- private BlockingQueue<PartQueueEntry> partQueue;
- private boolean writingToQueue, readersFinished;
- private boolean doKerberos, doAll;
- private List<String> rolesToImport, dbsToImport, tablesToImport, functionsToImport;
- private int parallel;
- private int batchSize;
-
- private HBaseImport() {}
-
- @VisibleForTesting
- public HBaseImport(String... args) throws ParseException {
- init(args);
- }
-
- private int init(String... args) throws ParseException {
- Options options = new Options();
-
- doAll = doKerberos = false;
- parallel = 1;
- batchSize = 1000;
-
- options.addOption(OptionBuilder
- .withLongOpt("all")
- .withDescription("Import the full metastore")
- .create('a'));
-
- options.addOption(OptionBuilder
- .withLongOpt("batchsize")
- .withDescription("Number of partitions to read and write in a batch, defaults to 1000")
- .hasArg()
- .create('b'));
-
- options.addOption(OptionBuilder
- .withLongOpt("database")
- .withDescription("Import a single database")
- .hasArgs()
- .create('d'));
-
- options.addOption(OptionBuilder
- .withLongOpt("help")
- .withDescription("You're looking at it")
- .create('h'));
-
- options.addOption(OptionBuilder
- .withLongOpt("function")
- .withDescription("Import a single function")
- .hasArgs()
- .create('f'));
-
- options.addOption(OptionBuilder
- .withLongOpt("kerberos")
- .withDescription("Import all kerberos related objects (master key, tokens)")
- .create('k'));
-
- options.addOption(OptionBuilder
- .withLongOpt("parallel")
- .withDescription("Parallel factor for loading (only applied to tables and partitions), " +
- "defaults to 1")
- .hasArg()
- .create('p'));
-
- options.addOption(OptionBuilder
- .withLongOpt("role")
- .withDescription("Import a single role")
- .hasArgs()
- .create('r'));
-
- options.addOption(OptionBuilder
- .withLongOpt("tables")
- .withDescription("Import a single tables")
- .hasArgs()
- .create('t'));
-
- CommandLine cli = new GnuParser().parse(options, args);
-
- // Process help, if it was asked for, this must be done first
- if (cli.hasOption('h')) {
- printHelp(options);
- return 1;
- }
-
- boolean hasCmd = false;
- // Now process the other command line args
- if (cli.hasOption('a')) {
- hasCmd = true;
- doAll = true;
- }
- if (cli.hasOption('b')) {
- batchSize = Integer.parseInt(cli.getOptionValue('b'));
- }
- if (cli.hasOption('d')) {
- hasCmd = true;
- dbsToImport = Arrays.asList(cli.getOptionValues('d'));
- }
- if (cli.hasOption('f')) {
- hasCmd = true;
- functionsToImport = Arrays.asList(cli.getOptionValues('f'));
- }
- if (cli.hasOption('p')) {
- parallel = Integer.parseInt(cli.getOptionValue('p'));
- }
- if (cli.hasOption('r')) {
- hasCmd = true;
- rolesToImport = Arrays.asList(cli.getOptionValues('r'));
- }
- if (cli.hasOption('k')) {
- doKerberos = true;
- }
- if (cli.hasOption('t')) {
- hasCmd = true;
- tablesToImport = Arrays.asList(cli.getOptionValues('t'));
- }
- if (!hasCmd) {
- printHelp(options);
- return 1;
- }
-
- dbs = new ArrayList<>();
- // We don't want to bound the size of the table queue because we keep it all in memory
- partitionedTables = new LinkedBlockingQueue<>();
- tableNameQueue = new LinkedBlockingQueue<>();
- indexNameQueue = new LinkedBlockingQueue<>();
-
- // Bound the size of this queue so we don't get too much in memory.
- partQueue = new ArrayBlockingQueue<>(parallel * 2);
- return 0;
- }
-
- private void printHelp(Options options) {
- (new HelpFormatter()).printHelp("hbaseschematool", options);
- }
-
- @VisibleForTesting
- void run() throws MetaException, InstantiationException, IllegalAccessException,
- NoSuchObjectException, InvalidObjectException, InterruptedException {
- // Order here is crucial, as you can't add tables until you've added databases, etc.
- init();
- if (doAll || rolesToImport != null) {
- copyRoles();
- }
- if (doAll || dbsToImport != null) {
- copyDbs();
- }
- if (doAll || dbsToImport != null || tablesToImport != null) {
- copyTables();
- copyPartitions();
- copyIndexes();
- }
- if (doAll || dbsToImport != null || functionsToImport != null) {
- copyFunctions();
- }
- if (doAll || doKerberos) {
- copyKerberos();
- }
- }
-
- private void init() throws MetaException, IllegalAccessException, InstantiationException {
- if (rdbmsConf != null) {
- // We've been configured for testing, so don't do anything here.
- return;
- }
- // We're depending on having everything properly in the path
- rdbmsConf = new HiveConf();
- hbaseConf = new HiveConf();//
- HiveConf.setVar(hbaseConf, HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
- HBaseStore.class.getName());
- HiveConf.setBoolVar(hbaseConf, HiveConf.ConfVars.METASTORE_FASTPATH, true);
-
- // First get a connection to the RDBMS based store
- rdbmsStore.get().setConf(rdbmsConf);
-
- // Get a connection to the HBase based store
- hbaseStore.get().setConf(hbaseConf);
- }
-
- private void copyRoles() throws NoSuchObjectException, InvalidObjectException, MetaException {
- screen("Copying roles");
- List<String> toCopy = doAll ? rdbmsStore.get().listRoleNames() : rolesToImport;
- for (String roleName : toCopy) {
- Role role = rdbmsStore.get().getRole(roleName);
- screen("Copying role " + roleName);
- hbaseStore.get().addRole(roleName, role.getOwnerName());
- }
- }
-
- private void copyDbs() throws MetaException, NoSuchObjectException, InvalidObjectException {
- screen("Copying databases");
- List<String> toCopy = doAll ? rdbmsStore.get().getAllDatabases() : dbsToImport;
- for (String dbName : toCopy) {
- Database db = rdbmsStore.get().getDatabase(dbName);
- dbs.add(db);
- screen("Copying database " + dbName);
- hbaseStore.get().createDatabase(db);
- }
- }
-
- private void copyTables() throws MetaException, InvalidObjectException, InterruptedException {
- screen("Copying tables");
-
- // Start the parallel threads that will copy the tables
- Thread[] copiers = new Thread[parallel];
- writingToQueue = true;
- for (int i = 0; i < parallel; i++) {
- copiers[i] = new TableCopier();
- copiers[i].start();
- }
-
- // Put tables from the databases we copied into the queue
- for (Database db : dbs) {
- screen("Coyping tables in database " + db.getName());
- for (String tableName : rdbmsStore.get().getAllTables(db.getName())) {
- tableNameQueue.put(new String[]{db.getName(), tableName});
- }
- }
-
- // Now put any specifically requested tables into the queue
- if (tablesToImport != null) {
- for (String compoundTableName : tablesToImport) {
- String[] tn = compoundTableName.split("\\.");
- if (tn.length != 2) {
- error(compoundTableName + " not in proper form. Must be in form dbname.tablename. " +
- "Ignoring this table and continuing.");
- } else {
- tableNameQueue.put(new String[]{tn[0], tn[1]});
- }
- }
- }
- writingToQueue = false;
-
- // Wait until we've finished adding all the tables
- for (Thread copier : copiers) copier.join();
- }
-
- private class TableCopier extends Thread {
- @Override
- public void run() {
- while (writingToQueue || tableNameQueue.size() > 0) {
- try {
- String[] name = tableNameQueue.poll(1, TimeUnit.SECONDS);
- if (name != null) {
- Table table = rdbmsStore.get().getTable(name[0], name[1]);
- // If this has partitions, put it in the list to fetch partions for
- if (table.getPartitionKeys() != null && table.getPartitionKeys().size() > 0) {
- partitionedTables.put(table);
- }
- screen("Copying table " + name[0] + "." + name[1]);
- hbaseStore.get().createTable(table);
-
- // See if the table has any constraints, and if so copy those as well
- List<SQLPrimaryKey> pk =
- rdbmsStore.get().getPrimaryKeys(table.getDbName(), table.getTableName());
- if (pk != null && pk.size() > 0) {
- LOG.debug("Found primary keys, adding them");
- hbaseStore.get().addPrimaryKeys(pk);
- }
-
- // Passing null as the target table name results in all of the foreign keys being
- // retrieved.
- List<SQLForeignKey> fks =
- rdbmsStore.get().getForeignKeys(null, null, table.getDbName(), table.getTableName());
- if (fks != null && fks.size() > 0) {
- LOG.debug("Found foreign keys, adding them");
- hbaseStore.get().addForeignKeys(fks);
- }
- }
- } catch (InterruptedException | MetaException | InvalidObjectException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- private void copyIndexes() throws MetaException, InvalidObjectException, InterruptedException {
- screen("Copying indexes");
-
- // Start the parallel threads that will copy the indexes
- Thread[] copiers = new Thread[parallel];
- writingToQueue = true;
- for (int i = 0; i < parallel; i++) {
- copiers[i] = new IndexCopier();
- copiers[i].start();
- }
-
- // Put indexes from the databases we copied into the queue
- for (Database db : dbs) {
- screen("Coyping indexes in database " + db.getName());
- for (String tableName : rdbmsStore.get().getAllTables(db.getName())) {
- for (Index index : rdbmsStore.get().getIndexes(db.getName(), tableName, -1)) {
- indexNameQueue.put(new String[]{db.getName(), tableName, index.getIndexName()});
- }
- }
- }
-
- // Now put any specifically requested tables into the queue
- if (tablesToImport != null) {
- for (String compoundTableName : tablesToImport) {
- String[] tn = compoundTableName.split("\\.");
- if (tn.length != 2) {
- error(compoundTableName + " not in proper form. Must be in form dbname.tablename. " +
- "Ignoring this table and continuing.");
- } else {
- for (Index index : rdbmsStore.get().getIndexes(tn[0], tn[1], -1)) {
- indexNameQueue.put(new String[]{tn[0], tn[1], index.getIndexName()});
- }
- }
- }
- }
-
- writingToQueue = false;
-
- // Wait until we've finished adding all the tables
- for (Thread copier : copiers) copier.join();
- }
-
- private class IndexCopier extends Thread {
- @Override
- public void run() {
- while (writingToQueue || indexNameQueue.size() > 0) {
- try {
- String[] name = indexNameQueue.poll(1, TimeUnit.SECONDS);
- if (name != null) {
- Index index = rdbmsStore.get().getIndex(name[0], name[1], name[2]);
- screen("Copying index " + name[0] + "." + name[1] + "." + name[2]);
- hbaseStore.get().addIndex(index);
- }
- } catch (InterruptedException | MetaException | InvalidObjectException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- /* Partition copying is a little complex. As we went through and copied the tables we put each
- * partitioned table into a queue. We will now go through that queue and add partitions for the
- * tables. We do the finding of partitions and writing of them separately and in parallel.
- * This way if there is one table with >> partitions then all of the others that skew won't
- * hurt us. To avoid pulling all of the partitions for a table into memory, we batch up
- * partitions (by default in batches of 1000) and copy them over in batches.
- */
- private void copyPartitions() throws MetaException, NoSuchObjectException,
- InvalidObjectException, InterruptedException {
- screen("Copying partitions");
- readersFinished = false;
- Thread[] readers = new Thread[parallel];
- Thread[] writers = new Thread[parallel];
- for (int i = 0; i < parallel; i++) {
- readers[i] = new PartitionReader();
- readers[i].start();
- writers[i] = new PartitionWriter();
- writers[i].start();
- }
-
- for (Thread reader : readers) reader.join();
- readersFinished = true;
-
- // Wait until we've finished adding all the partitions
- for (Thread writer : writers) writer.join();
- }
-
- private class PartitionReader extends Thread {
- @Override
- public void run() {
- while (partitionedTables.size() > 0) {
- try {
- Table table = partitionedTables.poll(1, TimeUnit.SECONDS);
- if (table != null) {
- screen("Fetching partitions for table " + table.getDbName() + "." +
- table.getTableName());
- List<String> partNames =
- rdbmsStore.get().listPartitionNames(table.getDbName(), table.getTableName(),
- (short) -1);
- if (partNames.size() <= batchSize) {
- LOG.debug("Adding all partition names to queue for " + table.getDbName() + "." +
- table.getTableName());
- partQueue.put(new PartQueueEntry(table.getDbName(), table.getTableName(), partNames));
- } else {
- int goUntil = partNames.size() % batchSize == 0 ? partNames.size() / batchSize :
- partNames.size() / batchSize + 1;
- for (int i = 0; i < goUntil; i++) {
- int start = i * batchSize;
- int end = Math.min((i + 1) * batchSize, partNames.size());
- LOG.debug("Adding partitions " + start + " to " + end + " for " + table.getDbName()
- + "." + table.getTableName());
- partQueue.put(new PartQueueEntry(table.getDbName(), table.getTableName(),
- partNames.subList(start, end)));
- }
- }
- }
- } catch (InterruptedException | MetaException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- private class PartitionWriter extends Thread {
- @Override
- public void run() {
- // This keeps us from throwing exceptions in our raw store calls
- Deadline.registerIfNot(1000000);
- while (!readersFinished || partQueue.size() > 0) {
- try {
- PartQueueEntry entry = partQueue.poll(1, TimeUnit.SECONDS);
- if (entry != null) {
- LOG.info("Writing partitions " + entry.dbName + "." + entry.tableName + "." +
- StringUtils.join(entry.partNames, ':'));
- // Fetch these partitions and write them to HBase
- Deadline.startTimer("hbaseimport");
- List<Partition> parts =
- rdbmsStore.get().getPartitionsByNames(entry.dbName, entry.tableName,
- entry.partNames);
- hbaseStore.get().addPartitions(entry.dbName, entry.tableName, parts);
- Deadline.stopTimer();
- }
- } catch (InterruptedException | MetaException | InvalidObjectException |
- NoSuchObjectException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- private void copyFunctions() throws MetaException, NoSuchObjectException, InvalidObjectException {
- screen("Copying functions");
- // Copy any functions from databases we copied.
- for (Database db : dbs) {
- screen("Copying functions in database " + db.getName());
- for (String funcName : rdbmsStore.get().getFunctions(db.getName(), "*")) {
- copyOneFunction(db.getName(), funcName);
- }
- }
- // Now do any specifically requested functions
- if (functionsToImport != null) {
- for (String compoundFuncName : functionsToImport) {
- String[] fn = compoundFuncName.split("\\.");
- if (fn.length != 2) {
- error(compoundFuncName + " not in proper form. Must be in form dbname.funcname. " +
- "Ignoring this function and continuing.");
- } else {
- copyOneFunction(fn[0], fn[1]);
- }
- }
- }
- }
-
- private void copyOneFunction(String dbName, String funcName) throws MetaException,
- InvalidObjectException {
- Function func = rdbmsStore.get().getFunction(dbName, funcName);
- screen("Copying function " + dbName + "." + funcName);
- hbaseStore.get().createFunction(func);
- }
-
- private void copyKerberos() throws MetaException {
- screen("Copying kerberos related items");
- for (String tokenId : rdbmsStore.get().getAllTokenIdentifiers()) {
- String token = rdbmsStore.get().getToken(tokenId);
- hbaseStore.get().addToken(tokenId, token);
- }
- for (String masterKey : rdbmsStore.get().getMasterKeys()) {
- hbaseStore.get().addMasterKey(masterKey);
- }
- }
-
- private void screen(String msg) {
- LOG.info(msg);
- System.out.println(msg);
- }
-
- private void error(String msg) {
- LOG.error(msg);
- System.err.println("ERROR: " + msg);
- }
-
- @VisibleForTesting
- void setConnections(RawStore rdbms, RawStore hbase) {
- rdbmsStore.set(rdbms);
- hbaseStore.set(hbase);
- rdbmsConf = rdbms.getConf();
- hbaseConf = hbase.getConf();
- }
-
- private static class PartQueueEntry {
- final String dbName;
- final String tableName;
- final List<String> partNames;
-
- PartQueueEntry(String d, String t, List<String> p) {
- dbName = d;
- tableName = t;
- partNames = p;
- }
- }
-
-}
[05/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
deleted file mode 100644
index e55c412..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DoubleColumnStatsAggregator.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implements
- IExtrapolatePartStatus {
-
- private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
-
- @Override
- public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
- List<ColumnStatistics> css) throws MetaException {
- ColumnStatisticsObj statsObj = null;
-
- // check if all the ColumnStatisticsObjs contain stats and all the ndv are
- // bitvectors
- boolean doAllPartitionContainStats = partNames.size() == css.size();
- LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
- NumDistinctValueEstimator ndvEstimator = null;
- String colType = null;
- for (ColumnStatistics cs : css) {
- if (cs.getStatsObjSize() != 1) {
- throw new MetaException(
- "The number of columns should be exactly one in aggrStats, but found "
- + cs.getStatsObjSize());
- }
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- if (statsObj == null) {
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
- .getStatsData().getSetField());
- }
- if (!cso.getStatsData().getDoubleStats().isSetBitVectors()
- || cso.getStatsData().getDoubleStats().getBitVectors().length() == 0) {
- ndvEstimator = null;
- break;
- } else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(cso.getStatsData().getDoubleStats().getBitVectors());
- if (ndvEstimator == null) {
- ndvEstimator = estimator;
- } else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
- break;
- }
- }
- }
- }
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- if (doAllPartitionContainStats || css.size() < 2) {
- DoubleColumnStatsData aggregateData = null;
- long lowerBound = 0;
- long higherBound = 0;
- double densityAvgSum = 0.0;
- for (ColumnStatistics cs : css) {
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
- lowerBound = Math.max(lowerBound, newData.getNumDVs());
- higherBound += newData.getNumDVs();
- densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
- if (ndvEstimator != null) {
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData
- .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- }
- }
- if (ndvEstimator != null) {
- // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
- // use uniform distribution assumption because we can merge bitvectors
- // to get a good estimation.
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- } else {
- long estimation;
- if (useDensityFunctionForNDVEstimation) {
- // We have estimation, lowerbound and higherbound. We use estimation
- // if it is between lowerbound and higherbound.
- double densityAvg = densityAvgSum / partNames.size();
- estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
- if (estimation < lowerBound) {
- estimation = lowerBound;
- } else if (estimation > higherBound) {
- estimation = higherBound;
- }
- } else {
- estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
- }
- aggregateData.setNumDVs(estimation);
- }
- columnStatisticsData.setDoubleStats(aggregateData);
- } else {
- // we need extrapolation
- LOG.debug("start extrapolation for " + colName);
- Map<String, Integer> indexMap = new HashMap<String, Integer>();
- for (int index = 0; index < partNames.size(); index++) {
- indexMap.put(partNames.get(index), index);
- }
- Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
- Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
- // while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
- double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
- // if not every partition uses bitvector for ndv, we just fall back to
- // the traditional extrapolation methods.
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
- }
- adjustedIndexMap.put(partName, (double) indexMap.get(partName));
- adjustedStatsMap.put(partName, cso.getStatsData());
- }
- } else {
- // we first merge all the adjacent bitvectors that we could merge and
- // derive new partition names and index.
- StringBuilder pseudoPartName = new StringBuilder();
- double pseudoIndexSum = 0;
- int length = 0;
- int curIndex = -1;
- DoubleColumnStatsData aggregateData = null;
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
- // newData.isSetBitVectors() should be true for sure because we
- // already checked it before.
- if (indexMap.get(partName) != curIndex) {
- // There is bitvector, but it is not adjacent to the previous ones.
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setDoubleStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
- }
- // reset everything
- pseudoPartName = new StringBuilder();
- pseudoIndexSum = 0;
- length = 0;
- ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- aggregateData = null;
- }
- curIndex = indexMap.get(partName);
- pseudoPartName.append(partName);
- pseudoIndexSum += curIndex;
- length++;
- curIndex++;
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
- newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setDoubleStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
- }
- }
- }
- extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
- adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
- }
- LOG.debug("Ndv estimatation for " + colName + " is "
- + columnStatisticsData.getDoubleStats().getNumDVs());
- statsObj.setStatsData(columnStatisticsData);
- return statsObj;
- }
-
- @Override
- public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
- int numPartsWithStats, Map<String, Double> adjustedIndexMap,
- Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- DoubleColumnStatsData extrapolateDoubleData = new DoubleColumnStatsData();
- Map<String, DoubleColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
- for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
- extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDoubleStats());
- }
- List<Map.Entry<String, DoubleColumnStatsData>> list = new LinkedList<Map.Entry<String, DoubleColumnStatsData>>(
- extractedAdjustedStatsMap.entrySet());
- // get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
- public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
- Map.Entry<String, DoubleColumnStatsData> o2) {
- return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
- }
- });
- double minInd = adjustedIndexMap.get(list.get(0).getKey());
- double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double lowValue = 0;
- double min = list.get(0).getValue().getLowValue();
- double max = list.get(list.size() - 1).getValue().getLowValue();
- if (minInd == maxInd) {
- lowValue = min;
- } else if (minInd < maxInd) {
- // left border is the min
- lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
- } else {
- // right border is the min
- lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
- }
-
- // get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
- public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
- Map.Entry<String, DoubleColumnStatsData> o2) {
- return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
- }
- });
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double highValue = 0;
- min = list.get(0).getValue().getHighValue();
- max = list.get(list.size() - 1).getValue().getHighValue();
- if (minInd == maxInd) {
- highValue = min;
- } else if (minInd < maxInd) {
- // right border is the max
- highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- highValue = (min + (max - min) * minInd / (minInd - maxInd));
- }
-
- // get the #nulls
- long numNulls = 0;
- for (Map.Entry<String, DoubleColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
- numNulls += entry.getValue().getNumNulls();
- }
- // we scale up sumNulls based on the number of partitions
- numNulls = numNulls * numParts / numPartsWithStats;
-
- // get the ndv
- long ndv = 0;
- long ndvMin = 0;
- long ndvMax = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
- public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
- Map.Entry<String, DoubleColumnStatsData> o2) {
- return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
- }
- });
- long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
- long higherBound = 0;
- for (Map.Entry<String, DoubleColumnStatsData> entry : list) {
- higherBound += entry.getValue().getNumDVs();
- }
- if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
- ndv = (long) ((highValue - lowValue) / densityAvg);
- if (ndv < lowerBound) {
- ndv = lowerBound;
- } else if (ndv > higherBound) {
- ndv = higherBound;
- }
- } else {
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- ndvMin = list.get(0).getValue().getNumDVs();
- ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
- if (minInd == maxInd) {
- ndv = ndvMin;
- } else if (minInd < maxInd) {
- // right border is the max
- ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
- }
- }
- extrapolateDoubleData.setLowValue(lowValue);
- extrapolateDoubleData.setHighValue(highValue);
- extrapolateDoubleData.setNumNulls(numNulls);
- extrapolateDoubleData.setNumDVs(ndv);
- extrapolateData.setDoubleStats(extrapolateDoubleData);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
deleted file mode 100644
index acf679e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/IExtrapolatePartStatus.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Map;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-
-public interface IExtrapolatePartStatus {
- // The following function will extrapolate the stats when the column stats of
- // some partitions are missing.
- /**
- * @param extrapolateData
- * it will carry back the specific stats, e.g., DOUBLE_STATS or
- * LONG_STATS
- * @param numParts
- * the total number of partitions
- * @param numPartsWithStats
- * the number of partitions that have stats
- * @param adjustedIndexMap
- * the partition name to index map
- * @param adjustedStatsMap
- * the partition name to its stats map
- * @param densityAvg
- * the average of ndv density, which is useful when
- * useDensityFunctionForNDVEstimation is true.
- */
- public abstract void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
- int numPartsWithStats, Map<String, Double> adjustedIndexMap,
- Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg);
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
deleted file mode 100644
index 2ee09f3..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/LongColumnStatsAggregator.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
- IExtrapolatePartStatus {
-
- private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
-
- @Override
- public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
- List<ColumnStatistics> css) throws MetaException {
- ColumnStatisticsObj statsObj = null;
-
- // check if all the ColumnStatisticsObjs contain stats and all the ndv are
- // bitvectors
- boolean doAllPartitionContainStats = partNames.size() == css.size();
- LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
- NumDistinctValueEstimator ndvEstimator = null;
- String colType = null;
- for (ColumnStatistics cs : css) {
- if (cs.getStatsObjSize() != 1) {
- throw new MetaException(
- "The number of columns should be exactly one in aggrStats, but found "
- + cs.getStatsObjSize());
- }
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- if (statsObj == null) {
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
- .getStatsData().getSetField());
- }
- if (!cso.getStatsData().getLongStats().isSetBitVectors()
- || cso.getStatsData().getLongStats().getBitVectors().length() == 0) {
- ndvEstimator = null;
- break;
- } else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(cso.getStatsData().getLongStats().getBitVectors());
- if (ndvEstimator == null) {
- ndvEstimator = estimator;
- } else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
- break;
- }
- }
- }
- }
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- if (doAllPartitionContainStats || css.size() < 2) {
- LongColumnStatsData aggregateData = null;
- long lowerBound = 0;
- long higherBound = 0;
- double densityAvgSum = 0.0;
- for (ColumnStatistics cs : css) {
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- LongColumnStatsData newData = cso.getStatsData().getLongStats();
- lowerBound = Math.max(lowerBound, newData.getNumDVs());
- higherBound += newData.getNumDVs();
- densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
- if (ndvEstimator != null) {
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData
- .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- }
- }
- if (ndvEstimator != null) {
- // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
- // use uniform distribution assumption because we can merge bitvectors
- // to get a good estimation.
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- } else {
- long estimation;
- if (useDensityFunctionForNDVEstimation) {
- // We have estimation, lowerbound and higherbound. We use estimation
- // if it is between lowerbound and higherbound.
- double densityAvg = densityAvgSum / partNames.size();
- estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
- if (estimation < lowerBound) {
- estimation = lowerBound;
- } else if (estimation > higherBound) {
- estimation = higherBound;
- }
- } else {
- estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
- }
- aggregateData.setNumDVs(estimation);
- }
- columnStatisticsData.setLongStats(aggregateData);
- } else {
- // we need extrapolation
- LOG.debug("start extrapolation for " + colName);
-
- Map<String, Integer> indexMap = new HashMap<String, Integer>();
- for (int index = 0; index < partNames.size(); index++) {
- indexMap.put(partNames.get(index), index);
- }
- Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
- Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
- // while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
- double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
- // if not every partition uses bitvector for ndv, we just fall back to
- // the traditional extrapolation methods.
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- LongColumnStatsData newData = cso.getStatsData().getLongStats();
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
- }
- adjustedIndexMap.put(partName, (double) indexMap.get(partName));
- adjustedStatsMap.put(partName, cso.getStatsData());
- }
- } else {
- // we first merge all the adjacent bitvectors that we could merge and
- // derive new partition names and index.
- StringBuilder pseudoPartName = new StringBuilder();
- double pseudoIndexSum = 0;
- int length = 0;
- int curIndex = -1;
- LongColumnStatsData aggregateData = null;
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- LongColumnStatsData newData = cso.getStatsData().getLongStats();
- // newData.isSetBitVectors() should be true for sure because we
- // already checked it before.
- if (indexMap.get(partName) != curIndex) {
- // There is bitvector, but it is not adjacent to the previous ones.
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setLongStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
- }
- // reset everything
- pseudoPartName = new StringBuilder();
- pseudoIndexSum = 0;
- length = 0;
- ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- aggregateData = null;
- }
- curIndex = indexMap.get(partName);
- pseudoPartName.append(partName);
- pseudoIndexSum += curIndex;
- length++;
- curIndex++;
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
- newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setLongStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
- }
- }
- }
- extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
- adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
- }
- statsObj.setStatsData(columnStatisticsData);
- LOG.debug("Ndv estimatation for " + colName + " is "
- + columnStatisticsData.getLongStats().getNumDVs());
- return statsObj;
- }
-
- @Override
- public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
- int numPartsWithStats, Map<String, Double> adjustedIndexMap,
- Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- LongColumnStatsData extrapolateLongData = new LongColumnStatsData();
- Map<String, LongColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
- for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
- extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getLongStats());
- }
- List<Map.Entry<String, LongColumnStatsData>> list = new LinkedList<Map.Entry<String, LongColumnStatsData>>(
- extractedAdjustedStatsMap.entrySet());
- // get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
- public int compare(Map.Entry<String, LongColumnStatsData> o1,
- Map.Entry<String, LongColumnStatsData> o2) {
- return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
- }
- });
- double minInd = adjustedIndexMap.get(list.get(0).getKey());
- double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long lowValue = 0;
- long min = list.get(0).getValue().getLowValue();
- long max = list.get(list.size() - 1).getValue().getLowValue();
- if (minInd == maxInd) {
- lowValue = min;
- } else if (minInd < maxInd) {
- // left border is the min
- lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
- } else {
- // right border is the min
- lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
- }
-
- // get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
- public int compare(Map.Entry<String, LongColumnStatsData> o1,
- Map.Entry<String, LongColumnStatsData> o2) {
- return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
- }
- });
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long highValue = 0;
- min = list.get(0).getValue().getHighValue();
- max = list.get(list.size() - 1).getValue().getHighValue();
- if (minInd == maxInd) {
- highValue = min;
- } else if (minInd < maxInd) {
- // right border is the max
- highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
- }
-
- // get the #nulls
- long numNulls = 0;
- for (Map.Entry<String, LongColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
- numNulls += entry.getValue().getNumNulls();
- }
- // we scale up sumNulls based on the number of partitions
- numNulls = numNulls * numParts / numPartsWithStats;
-
- // get the ndv
- long ndv = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
- public int compare(Map.Entry<String, LongColumnStatsData> o1,
- Map.Entry<String, LongColumnStatsData> o2) {
- return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
- }
- });
- long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
- long higherBound = 0;
- for (Map.Entry<String, LongColumnStatsData> entry : list) {
- higherBound += entry.getValue().getNumDVs();
- }
- if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
- ndv = (long) ((highValue - lowValue) / densityAvg);
- if (ndv < lowerBound) {
- ndv = lowerBound;
- } else if (ndv > higherBound) {
- ndv = higherBound;
- }
- } else {
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- min = list.get(0).getValue().getNumDVs();
- max = list.get(list.size() - 1).getValue().getNumDVs();
- if (minInd == maxInd) {
- ndv = min;
- } else if (minInd < maxInd) {
- // right border is the max
- ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
- }
- }
- extrapolateLongData.setLowValue(lowValue);
- extrapolateLongData.setHighValue(highValue);
- extrapolateLongData.setNumNulls(numNulls);
- extrapolateLongData.setNumDVs(ndv);
- extrapolateData.setLongStats(extrapolateLongData);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java
deleted file mode 100644
index 4c2d1bc..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BinaryColumnStatsMerger.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-
-public class BinaryColumnStatsMerger extends ColumnStatsMerger {
-
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats();
- BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats();
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java
deleted file mode 100644
index 8e50153..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/BooleanColumnStatsMerger.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-
-public class BooleanColumnStatsMerger extends ColumnStatsMerger {
-
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- BooleanColumnStatsData aggregateData = aggregateColStats.getStatsData().getBooleanStats();
- BooleanColumnStatsData newData = newColStats.getStatsData().getBooleanStats();
- aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
- aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java
deleted file mode 100644
index 474d4dd..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMerger.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class ColumnStatsMerger {
- protected final Logger LOG = LoggerFactory.getLogger(ColumnStatsMerger.class.getName());
-
- public abstract void merge(ColumnStatisticsObj aggregateColStats,
- ColumnStatisticsObj newColStats);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java
deleted file mode 100644
index 0ce1847..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/ColumnStatsMergerFactory.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-
-public class ColumnStatsMergerFactory {
-
- private ColumnStatsMergerFactory() {
- }
-
- public static ColumnStatsMerger getColumnStatsMerger(ColumnStatisticsObj statsObjNew,
- ColumnStatisticsObj statsObjOld) {
- ColumnStatsMerger agg;
- _Fields typeNew = statsObjNew.getStatsData().getSetField();
- _Fields typeOld = statsObjOld.getStatsData().getSetField();
- // make sure that they have the same type
- typeNew = typeNew == typeOld ? typeNew : null;
- switch (typeNew) {
- case BOOLEAN_STATS:
- agg = new BooleanColumnStatsMerger();
- break;
- case LONG_STATS: {
- agg = new LongColumnStatsMerger();
- break;
- }
- case DOUBLE_STATS: {
- agg = new DoubleColumnStatsMerger();
- break;
- }
- case STRING_STATS: {
- agg = new StringColumnStatsMerger();
- break;
- }
- case BINARY_STATS:
- agg = new BinaryColumnStatsMerger();
- break;
- case DECIMAL_STATS: {
- agg = new DecimalColumnStatsMerger();
- break;
- }
- case DATE_STATS: {
- agg = new DateColumnStatsMerger();
- break;
- }
- default:
- throw new IllegalArgumentException("Unknown stats type " + typeNew.toString());
- }
- return agg;
- }
-
- public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
- ColumnStatisticsObj cso = new ColumnStatisticsObj();
- ColumnStatisticsData csd = new ColumnStatisticsData();
- cso.setColName(colName);
- cso.setColType(colType);
- switch (type) {
- case BOOLEAN_STATS:
- csd.setBooleanStats(new BooleanColumnStatsData());
- break;
-
- case LONG_STATS:
- csd.setLongStats(new LongColumnStatsData());
- break;
-
- case DOUBLE_STATS:
- csd.setDoubleStats(new DoubleColumnStatsData());
- break;
-
- case STRING_STATS:
- csd.setStringStats(new StringColumnStatsData());
- break;
-
- case BINARY_STATS:
- csd.setBinaryStats(new BinaryColumnStatsData());
- break;
-
- case DECIMAL_STATS:
- csd.setDecimalStats(new DecimalColumnStatsData());
- break;
-
- case DATE_STATS:
- csd.setDateStats(new DateColumnStatsData());
- break;
-
- default:
- throw new IllegalArgumentException("Unknown stats type");
- }
-
- cso.setStatsData(csd);
- return cso;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java
deleted file mode 100644
index 2542a00..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DateColumnStatsMerger.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Date;
-import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
-
-public class DateColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DateColumnStatsData aggregateData = aggregateColStats.getStatsData().getDateStats();
- DateColumnStatsData newData = newColStats.getStatsData().getDateStats();
- Date lowValue = aggregateData.getLowValue().compareTo(newData.getLowValue()) < 0 ? aggregateData
- .getLowValue() : newData.getLowValue();
- aggregateData.setLowValue(lowValue);
- Date highValue = aggregateData.getHighValue().compareTo(newData.getHighValue()) >= 0 ? aggregateData
- .getHighValue() : newData.getHighValue();
- aggregateData.setHighValue(highValue);
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java
deleted file mode 100644
index 4e8e129..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DecimalColumnStatsMerger.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-
-public class DecimalColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DecimalColumnStatsData aggregateData = aggregateColStats.getStatsData().getDecimalStats();
- DecimalColumnStatsData newData = newColStats.getStatsData().getDecimalStats();
- Decimal lowValue = aggregateData.getLowValue() != null
- && (aggregateData.getLowValue().compareTo(newData.getLowValue()) > 0) ? aggregateData
- .getLowValue() : newData.getLowValue();
- aggregateData.setLowValue(lowValue);
- Decimal highValue = aggregateData.getHighValue() != null
- && (aggregateData.getHighValue().compareTo(newData.getHighValue()) > 0) ? aggregateData
- .getHighValue() : newData.getHighValue();
- aggregateData.setHighValue(highValue);
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java
deleted file mode 100644
index 4ef5c39..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/DoubleColumnStatsMerger.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-
-public class DoubleColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DoubleColumnStatsData aggregateData = aggregateColStats.getStatsData().getDoubleStats();
- DoubleColumnStatsData newData = newColStats.getStatsData().getDoubleStats();
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java
deleted file mode 100644
index acf7f03..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/LongColumnStatsMerger.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-
-public class LongColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- LongColumnStatsData aggregateData = aggregateColStats.getStatsData().getLongStats();
- LongColumnStatsData newData = newColStats.getStatsData().getLongStats();
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java
deleted file mode 100644
index b3cd33c..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/merge/StringColumnStatsMerger.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-
-public class StringColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- StringColumnStatsData aggregateData = aggregateColStats.getStatsData().getStringStats();
- StringColumnStatsData newData = newColStats.getStatsData().getStringStats();
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (!aggregateData.isSetBitVectors() || aggregateData.getBitVectors().length() == 0
- || !newData.isSetBitVectors() || newData.getBitVectors().length() == 0) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(aggregateData.getBitVectors());
- NumDistinctValueEstimator newEst = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors());
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setBitVectors(oldEst.serialize());
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
----------------------------------------------------------------------
diff --git a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto b/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
deleted file mode 100644
index 53c381b..0000000
--- a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto
+++ /dev/null
@@ -1,368 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-enum PrincipalType {
- USER = 0;
- ROLE = 1;
-}
-
-message AggrStats {
- required int64 parts_found = 1;
- repeated ColumnStats col_stats = 2;
-}
-
-message AggrStatsBloomFilter {
- message BloomFilter {
- required int32 num_bits = 1;
- required int32 num_funcs = 2;
- repeated int64 bits = 3;
- }
- required bytes db_name = 1;
- required bytes table_name = 2;
- required BloomFilter bloom_filter = 3;
- required int64 aggregated_at = 4;
-}
-
-message AggrStatsInvalidatorFilter {
- message Entry {
- required bytes db_name = 1;
- required bytes table_name = 2;
- required bytes part_name = 3;
- }
-
- repeated Entry to_invalidate = 1;
- required int64 run_every = 2;
- required int64 max_cache_entry_life = 3;
-}
-
-message ColumnStats {
-
- message BooleanStats {
- optional int64 num_trues = 1;
- optional int64 num_falses = 2;
- }
-
- message LongStats {
- optional sint64 low_value = 1;
- optional sint64 high_value = 2;
- }
-
- message DoubleStats {
- optional double low_value = 1;
- optional double high_value = 2;
- }
-
- message StringStats {
- optional int64 max_col_length = 1;
- optional double avg_col_length = 2;
- }
-
- message DecimalStats {
- message Decimal {
- required bytes unscaled = 1;
- required int32 scale = 2;
- }
- optional Decimal low_value = 1;
- optional Decimal high_value = 2;
- }
-
- optional int64 last_analyzed = 1;
- required string column_type = 2;
- optional int64 num_nulls = 3;
- optional int64 num_distinct_values = 4;
- optional BooleanStats bool_stats = 5;
- optional LongStats long_stats = 6;
- optional DoubleStats double_stats = 7;
- optional StringStats string_stats = 8;
- optional StringStats binary_stats = 9;
- optional DecimalStats decimal_stats = 10;
- optional string column_name = 11;
- optional string bit_vectors = 12;
-}
-
-message Database {
- optional string description = 1;
- optional string uri = 2;
- optional Parameters parameters = 3;
- optional PrincipalPrivilegeSet privileges = 4;
- optional string owner_name = 5;
- optional PrincipalType owner_type = 6;
-}
-
-message DelegationToken {
- required string token_str = 1;
-}
-
-message FieldSchema {
- required string name = 1;
- required string type = 2;
- optional string comment = 3;
-}
-
-message Function {
- enum FunctionType {
- JAVA = 1;
- }
-
- message ResourceUri {
- enum ResourceType {
- JAR = 1;
- FILE = 2;
- ARCHIVE = 3;
- }
- required ResourceType resource_type = 1;
- required string uri = 2;
- }
-
- optional string class_name = 1;
- optional string owner_name = 2;
- optional PrincipalType owner_type = 3;
- optional sint64 create_time = 4;
- optional FunctionType function_type = 5;
- repeated ResourceUri resource_uris = 6;
-}
-
-message MasterKey {
- required string master_key = 1;
-}
-
-message ParameterEntry {
- required string key = 1;
- required string value = 2;
-}
-
-message Parameters {
- repeated ParameterEntry parameter = 1;
-}
-
-message Partition {
- optional int64 create_time = 1;
- optional int64 last_access_time = 2;
- optional string location = 3;
- optional Parameters sd_parameters = 4; // storage descriptor parameters
- required bytes sd_hash = 5;
- optional Parameters parameters = 6; // partition parameters
- // We don't support partition level privileges
-}
-
-message PrincipalPrivilegeSetEntry {
- required string principal_name = 1;
- repeated PrivilegeGrantInfo privileges = 2;
-}
-
-message PrincipalPrivilegeSet {
- repeated PrincipalPrivilegeSetEntry users = 1;
- repeated PrincipalPrivilegeSetEntry roles = 2;
-}
-
-message PrivilegeGrantInfo {
- optional string privilege = 1;
- optional int64 create_time = 2;
- optional string grantor = 3;
- optional PrincipalType grantor_type = 4;
- optional bool grant_option = 5;
-}
-
-message RoleGrantInfo {
- required string principal_name = 1;
- required PrincipalType principal_type = 2;
- optional int64 add_time = 3;
- optional string grantor = 4;
- optional PrincipalType grantor_type = 5;
- optional bool grant_option = 6;
-}
-
-message RoleGrantInfoList {
- repeated RoleGrantInfo grant_info = 1;
-}
-
-message RoleList {
- repeated string role = 1;
-}
-
-message Role {
- optional int64 create_time = 1;
- optional string owner_name = 2;
-}
-
-message StorageDescriptor {
- message Order {
- required string column_name = 1;
- optional sint32 order = 2 [default = 1];
- }
-
- message SerDeInfo {
- optional string name = 1;
- optional string serialization_lib = 2;
- optional Parameters parameters = 3;
- }
-
- message SkewedInfo {
- message SkewedColValueList {
- repeated string skewed_col_value = 1;
- }
-
- message SkewedColValueLocationMap {
- repeated string key = 1;
- required string value = 2;
- }
-
- repeated string skewed_col_names = 1;
- repeated SkewedColValueList skewed_col_values = 2;
- repeated SkewedColValueLocationMap skewed_col_value_location_maps = 3;
- }
-
- repeated FieldSchema cols = 1;
- optional string input_format = 2;
- optional string output_format = 3;
- optional bool is_compressed = 4;
- optional sint32 num_buckets = 5;
- optional SerDeInfo serde_info = 6;
- repeated string bucket_cols = 7;
- repeated Order sort_cols = 8;
- optional SkewedInfo skewed_info = 9;
- optional bool stored_as_sub_directories = 10;
-}
-
-message Table {
- optional string owner = 1;
- optional int64 create_time = 2;
- optional int64 last_access_time = 3;
- optional int64 retention = 4;
- optional string location = 5;
- optional Parameters sd_parameters = 6; // storage descriptor parameters
- required bytes sd_hash = 7;
- repeated FieldSchema partition_keys = 8;
- optional Parameters parameters = 9;
- optional string view_original_text = 10;
- optional string view_expanded_text = 11;
- optional string table_type = 12;
- optional PrincipalPrivilegeSet privileges = 13;
- optional bool is_temporary = 14;
- optional bool is_rewrite_enabled = 15;
-}
-
-message Index {
- optional string indexHandlerClass = 1; // reserved
- required string dbName = 2;
- required string origTableName = 3;
- optional string location = 4;
- optional Parameters sd_parameters = 5; // storage descriptor parameters
- optional int32 createTime = 6;
- optional int32 lastAccessTime = 7;
- optional string indexTableName = 8;
- optional bytes sd_hash = 9;
- optional Parameters parameters = 10;
- optional bool deferredRebuild = 11;
-}
-
-message PartitionKeyComparator {
- required string names = 1;
- required string types = 2;
- message Mark {
- required string value = 1;
- required bool inclusive = 2;
- }
- message Range {
- required string key = 1;
- optional Mark start = 2;
- optional Mark end = 3;
- }
- message Operator {
- enum Type {
- LIKE = 0;
- NOTEQUALS = 1;
- }
- required Type type = 1;
- required string key = 2;
- required string val = 3;
- }
- repeated Operator op = 3;
- repeated Range range = 4;
-}
-
-message PrimaryKey {
- message PrimaryKeyColumn {
- required string column_name = 1;
- required sint32 key_seq = 2;
- }
-
- required string pk_name = 1;
- repeated PrimaryKeyColumn cols = 2;
- optional bool enable_constraint = 3;
- optional bool validate_constraint = 4;
- optional bool rely_constraint = 5;
-}
-
-message ForeignKeys {
- message ForeignKey {
- message ForeignKeyColumn {
- required string column_name = 1;
- required string referenced_column_name = 2;
- required sint32 key_seq = 3;
- }
-
- required string fk_name = 1;
- required string referenced_db_name = 2;
- required string referenced_table_name = 3;
- optional string referenced_pk_name = 4;
- optional int32 update_rule = 5;
- optional int32 delete_rule = 6;
- repeated ForeignKeyColumn cols = 7;
- optional bool enable_constraint = 8;
- optional bool validate_constraint = 9;
- optional bool rely_constraint = 10;
- }
-
- repeated ForeignKey fks = 1;
-}
-
-message UniqueConstraints {
- message UniqueConstraint {
- message UniqueConstraintColumn {
- required string column_name = 1;
- required sint32 key_seq = 2;
- }
-
- required string uk_name = 1;
- repeated UniqueConstraintColumn cols = 2;
- optional bool enable_constraint = 3;
- optional bool validate_constraint = 4;
- optional bool rely_constraint = 5;
- }
-
- repeated UniqueConstraint uks = 1;
-}
-
-message NotNullConstraints {
- message NotNullConstraint {
- message NotNullConstraintColumn {
- required string column_name = 1;
- }
-
- required string nn_name = 1;
- repeated NotNullConstraintColumn cols = 2;
- optional bool enable_constraint = 3;
- optional bool validate_constraint = 4;
- optional bool rely_constraint = 5;
- }
-
- repeated NotNullConstraint nns = 1;
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
deleted file mode 100644
index 784648a..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.FileFormatProxy;
-import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-
-/**
- * Mock utilities for HBaseStore testing
- */
-public class MockUtils {
-
- /**
- * The default impl is in ql package and is not available in unit tests.
- */
- public static class NOOPProxy implements PartitionExpressionProxy {
-
- @Override
- public String convertExprToFilter(byte[] expr) throws MetaException {
- return null;
- }
-
- @Override
- public boolean filterPartitionsByExpr(List<String> partColumnNames,
- List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr, String defaultPartitionName,
- List<String> partitionNames) throws MetaException {
- return false;
- }
-
- @Override
- public SearchArgument createSarg(byte[] expr) {
- return null;
- }
-
- @Override
- public FileMetadataExprType getMetadataType(String inputFormat) {
- return null;
- }
-
- @Override
- public FileFormatProxy getFileFormatProxy(FileMetadataExprType type) {
- return null;
- }
- }
-
- static HBaseStore init(Configuration conf, HTableInterface htable,
- final SortedMap<String, Cell> rows) throws IOException {
- ((HiveConf)conf).setVar(ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, NOOPProxy.class.getName());
- Mockito.when(htable.get(Mockito.any(Get.class))).thenAnswer(new Answer<Result>() {
- @Override
- public Result answer(InvocationOnMock invocation) throws Throwable {
- Get get = (Get) invocation.getArguments()[0];
- Cell cell = rows.get(new String(get.getRow()));
- if (cell == null) {
- return new Result();
- } else {
- return Result.create(new Cell[]{cell});
- }
- }
- });
-
- Mockito.when(htable.get(Mockito.anyListOf(Get.class))).thenAnswer(new Answer<Result[]>() {
- @Override
- public Result[] answer(InvocationOnMock invocation) throws Throwable {
- @SuppressWarnings("unchecked")
- List<Get> gets = (List<Get>) invocation.getArguments()[0];
- Result[] results = new Result[gets.size()];
- for (int i = 0; i < gets.size(); i++) {
- Cell cell = rows.get(new String(gets.get(i).getRow()));
- Result result;
- if (cell == null) {
- result = new Result();
- } else {
- result = Result.create(new Cell[]{cell});
- }
- results[i] = result;
- }
- return results;
- }
- });
-
- Mockito.when(htable.getScanner(Mockito.any(Scan.class))).thenAnswer(new Answer<ResultScanner>() {
- @Override
- public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
- Scan scan = (Scan)invocation.getArguments()[0];
- List<Result> results = new ArrayList<Result>();
- String start = new String(scan.getStartRow());
- String stop = new String(scan.getStopRow());
- SortedMap<String, Cell> sub = rows.subMap(start, stop);
- for (Map.Entry<String, Cell> e : sub.entrySet()) {
- results.add(Result.create(new Cell[]{e.getValue()}));
- }
-
- final Iterator<Result> iter = results.iterator();
-
- return new ResultScanner() {
- @Override
- public Result next() throws IOException {
- return null;
- }
-
- @Override
- public Result[] next(int nbRows) throws IOException {
- return new Result[0];
- }
-
- @Override
- public void close() {
-
- }
-
- @Override
- public Iterator<Result> iterator() {
- return iter;
- }
- };
- }
- });
-
- Mockito.doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- Put put = (Put)invocation.getArguments()[0];
- rows.put(new String(put.getRow()), put.getFamilyCellMap().firstEntry().getValue().get(0));
- return null;
- }
- }).when(htable).put(Mockito.any(Put.class));
-
- Mockito.when(htable.checkAndPut(Mockito.any(byte[].class), Mockito.any(byte[].class),
- Mockito.any(byte[].class), Mockito.any(byte[].class), Mockito.any(Put.class))).thenAnswer(
- new Answer<Boolean>() {
-
- @Override
- public Boolean answer(InvocationOnMock invocation) throws Throwable {
- // Always say it succeeded and overwrite
- Put put = (Put)invocation.getArguments()[4];
- rows.put(new String(put.getRow()),
- put.getFamilyCellMap().firstEntry().getValue().get(0));
- return true;
- }
- });
-
- Mockito.doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- Delete del = (Delete)invocation.getArguments()[0];
- rows.remove(new String(del.getRow()));
- return null;
- }
- }).when(htable).delete(Mockito.any(Delete.class));
-
- Mockito.when(htable.checkAndDelete(Mockito.any(byte[].class), Mockito.any(byte[].class),
- Mockito.any(byte[].class), Mockito.any(byte[].class), Mockito.any(Delete.class))).thenAnswer(
- new Answer<Boolean>() {
-
- @Override
- public Boolean answer(InvocationOnMock invocation) throws Throwable {
- // Always say it succeeded
- Delete del = (Delete)invocation.getArguments()[4];
- rows.remove(new String(del.getRow()));
- return true;
- }
- });
-
- // Mock connection
- HBaseConnection hconn = Mockito.mock(HBaseConnection.class);
- Mockito.when(hconn.getHBaseTable(Mockito.anyString())).thenReturn(htable);
- HiveConf.setVar(conf, HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS, HBaseReadWrite.TEST_CONN);
- HBaseReadWrite.setTestConnection(hconn);
- HBaseReadWrite.setConf(conf);
- HBaseStore store = new HBaseStore();
- store.setConf(conf);
- return store;
- }
-}
[12/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index edfbf3a..48602bd 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -22,11 +22,16 @@ import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.URL;
import java.net.URLClassLoader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -37,6 +42,10 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -46,6 +55,9 @@ import com.google.common.collect.Maps;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.ListUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.metastore.api.Decimal;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SkewedInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -114,6 +126,7 @@ public class MetaStoreUtils {
// configuration parameter documentation
// HIVE_SUPPORT_SPECICAL_CHARACTERS_IN_TABLE_NAMES in HiveConf as well.
public static final char[] specialCharactersInTableNames = new char[] { '/' };
+ final static Charset ENCODING = StandardCharsets.UTF_8;
public static Table createColumnsetSchema(String name, List<String> columns,
List<String> partCols, Configuration conf) throws MetaException {
@@ -1988,4 +2001,88 @@ public class MetaStoreUtils {
return colStats;
}
+ /**
+ * Produce a hash for the storage descriptor
+ * @param sd storage descriptor to hash
+ * @param md message descriptor to use to generate the hash
+ * @return the hash as a byte array
+ */
+ public static byte[] hashStorageDescriptor(StorageDescriptor sd, MessageDigest md) {
+ // Note all maps and lists have to be absolutely sorted. Otherwise we'll produce different
+ // results for hashes based on the OS or JVM being used.
+ md.reset();
+ for (FieldSchema fs : sd.getCols()) {
+ md.update(fs.getName().getBytes(ENCODING));
+ md.update(fs.getType().getBytes(ENCODING));
+ if (fs.getComment() != null) md.update(fs.getComment().getBytes(ENCODING));
+ }
+ if (sd.getInputFormat() != null) {
+ md.update(sd.getInputFormat().getBytes(ENCODING));
+ }
+ if (sd.getOutputFormat() != null) {
+ md.update(sd.getOutputFormat().getBytes(ENCODING));
+ }
+ md.update(sd.isCompressed() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
+ md.update(Integer.toString(sd.getNumBuckets()).getBytes(ENCODING));
+ if (sd.getSerdeInfo() != null) {
+ SerDeInfo serde = sd.getSerdeInfo();
+ if (serde.getName() != null) {
+ md.update(serde.getName().getBytes(ENCODING));
+ }
+ if (serde.getSerializationLib() != null) {
+ md.update(serde.getSerializationLib().getBytes(ENCODING));
+ }
+ if (serde.getParameters() != null) {
+ SortedMap<String, String> params = new TreeMap<>(serde.getParameters());
+ for (Entry<String, String> param : params.entrySet()) {
+ md.update(param.getKey().getBytes(ENCODING));
+ md.update(param.getValue().getBytes(ENCODING));
+ }
+ }
+ }
+ if (sd.getBucketCols() != null) {
+ List<String> bucketCols = new ArrayList<>(sd.getBucketCols());
+ for (String bucket : bucketCols) md.update(bucket.getBytes(ENCODING));
+ }
+ if (sd.getSortCols() != null) {
+ SortedSet<Order> orders = new TreeSet<>(sd.getSortCols());
+ for (Order order : orders) {
+ md.update(order.getCol().getBytes(ENCODING));
+ md.update(Integer.toString(order.getOrder()).getBytes(ENCODING));
+ }
+ }
+ if (sd.getSkewedInfo() != null) {
+ SkewedInfo skewed = sd.getSkewedInfo();
+ if (skewed.getSkewedColNames() != null) {
+ SortedSet<String> colnames = new TreeSet<>(skewed.getSkewedColNames());
+ for (String colname : colnames) md.update(colname.getBytes(ENCODING));
+ }
+ if (skewed.getSkewedColValues() != null) {
+ SortedSet<String> sortedOuterList = new TreeSet<>();
+ for (List<String> innerList : skewed.getSkewedColValues()) {
+ SortedSet<String> sortedInnerList = new TreeSet<>(innerList);
+ sortedOuterList.add(StringUtils.join(sortedInnerList, "."));
+ }
+ for (String colval : sortedOuterList) md.update(colval.getBytes(ENCODING));
+ }
+ if (skewed.getSkewedColValueLocationMaps() != null) {
+ SortedMap<String, String> sortedMap = new TreeMap<>();
+ for (Entry<List<String>, String> smap : skewed.getSkewedColValueLocationMaps().entrySet()) {
+ SortedSet<String> sortedKey = new TreeSet<>(smap.getKey());
+ sortedMap.put(StringUtils.join(sortedKey, "."), smap.getValue());
+ }
+ for (Entry<String, String> e : sortedMap.entrySet()) {
+ md.update(e.getKey().getBytes(ENCODING));
+ md.update(e.getValue().getBytes(ENCODING));
+ }
+ }
+ md.update(sd.isStoredAsSubDirectories() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
+ }
+
+ return md.digest();
+ }
+
+ public static double decimalToDouble(Decimal decimal) {
+ return new BigDecimal(new BigInteger(decimal.getUnscaled()), decimal.getScale()).doubleValue();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
index b6fb4fd..80b17e0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.StatObjectConverter;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
@@ -40,7 +41,6 @@ import org.apache.hadoop.hive.metastore.api.TableMeta;
import org.apache.hadoop.hive.metastore.cache.CachedStore.PartitionWrapper;
import org.apache.hadoop.hive.metastore.cache.CachedStore.StorageDescriptorWrapper;
import org.apache.hadoop.hive.metastore.cache.CachedStore.TableWrapper;
-import org.apache.hadoop.hive.metastore.hbase.HBaseUtils;
import org.apache.hive.common.util.HiveStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +117,7 @@ public class SharedCache {
}
TableWrapper wrapper;
if (tbl.getSd() != null) {
- byte[] sdHash = HBaseUtils.hashStorageDescriptor(tbl.getSd(), md);
+ byte[] sdHash = MetaStoreUtils.hashStorageDescriptor(tbl.getSd(), md);
StorageDescriptor sd = tbl.getSd();
increSd(sd, sdHash);
tblCopy.setSd(null);
@@ -285,7 +285,7 @@ public class SharedCache {
Partition partCopy = part.deepCopy();
PartitionWrapper wrapper;
if (part.getSd()!=null) {
- byte[] sdHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
+ byte[] sdHash = MetaStoreUtils.hashStorageDescriptor(part.getSd(), md);
StorageDescriptor sd = part.getSd();
increSd(sd, sdHash);
partCopy.setSd(null);
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
new file mode 100644
index 0000000..e6c836b
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+public class BinaryColumnStatsAggregator extends ColumnStatsAggregator {
+
+ @Override
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+ BinaryColumnStatsData aggregateData = null;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ BinaryColumnStatsData newData = cso.getStatsData().getBinaryStats();
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
+ aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ }
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ columnStatisticsData.setBinaryStats(aggregateData);
+ statsObj.setStatsData(columnStatisticsData);
+ return statsObj;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
new file mode 100644
index 0000000..a34bc9f
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+public class BooleanColumnStatsAggregator extends ColumnStatsAggregator {
+
+ @Override
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+ BooleanColumnStatsData aggregateData = null;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ BooleanColumnStatsData newData = cso.getStatsData().getBooleanStats();
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
+ aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ }
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ columnStatisticsData.setBooleanStats(aggregateData);
+ statsObj.setStatsData(columnStatisticsData);
+ return statsObj;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
new file mode 100644
index 0000000..a52e5e5
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+public abstract class ColumnStatsAggregator {
+ public boolean useDensityFunctionForNDVEstimation;
+ public double ndvTuner;
+ public abstract ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java
new file mode 100644
index 0000000..173e06f
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+
+public class ColumnStatsAggregatorFactory {
+
+ private ColumnStatsAggregatorFactory() {
+ }
+
+ public static ColumnStatsAggregator getColumnStatsAggregator(_Fields type,
+ boolean useDensityFunctionForNDVEstimation, double ndvTuner) {
+ ColumnStatsAggregator agg;
+ switch (type) {
+ case BOOLEAN_STATS:
+ agg = new BooleanColumnStatsAggregator();
+ break;
+ case LONG_STATS:
+ agg = new LongColumnStatsAggregator();
+ break;
+ case DATE_STATS:
+ agg = new DateColumnStatsAggregator();
+ break;
+ case DOUBLE_STATS:
+ agg = new DoubleColumnStatsAggregator();
+ break;
+ case STRING_STATS:
+ agg = new StringColumnStatsAggregator();
+ break;
+ case BINARY_STATS:
+ agg = new BinaryColumnStatsAggregator();
+ break;
+ case DECIMAL_STATS:
+ agg = new DecimalColumnStatsAggregator();
+ break;
+ default:
+ throw new RuntimeException("Woh, bad. Unknown stats type " + type.toString());
+ }
+ agg.useDensityFunctionForNDVEstimation = useDensityFunctionForNDVEstimation;
+ agg.ndvTuner = ndvTuner;
+ return agg;
+ }
+
+ public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ cso.setColName(colName);
+ cso.setColType(colType);
+ switch (type) {
+ case BOOLEAN_STATS:
+ csd.setBooleanStats(new BooleanColumnStatsData());
+ break;
+
+ case LONG_STATS:
+ csd.setLongStats(new LongColumnStatsData());
+ break;
+
+ case DATE_STATS:
+ csd.setDateStats(new DateColumnStatsData());
+ break;
+
+ case DOUBLE_STATS:
+ csd.setDoubleStats(new DoubleColumnStatsData());
+ break;
+
+ case STRING_STATS:
+ csd.setStringStats(new StringColumnStatsData());
+ break;
+
+ case BINARY_STATS:
+ csd.setBinaryStats(new BinaryColumnStatsData());
+ break;
+
+ case DECIMAL_STATS:
+ csd.setDecimalStats(new DecimalColumnStatsData());
+ break;
+
+ default:
+ throw new RuntimeException("Woh, bad. Unknown stats type!");
+ }
+
+ cso.setStatsData(csd);
+ return cso;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java
new file mode 100644
index 0000000..c5e72eb
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java
@@ -0,0 +1,371 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.StatObjectConverter;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implements
+ IExtrapolatePartStatus {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DecimalColumnStatsAggregator.class);
+
+ @Override
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+
+ // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+ // bitvectors
+ boolean doAllPartitionContainStats = partNames.size() == css.size();
+ LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
+ NumDistinctValueEstimator ndvEstimator = null;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ if (!cso.getStatsData().getDecimalStats().isSetBitVectors()
+ || cso.getStatsData().getDecimalStats().getBitVectors().length() == 0) {
+ ndvEstimator = null;
+ break;
+ } else {
+ // check if all of the bit vectors can merge
+ NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(cso.getStatsData().getDecimalStats().getBitVectors());
+ if (ndvEstimator == null) {
+ ndvEstimator = estimator;
+ } else {
+ if (ndvEstimator.canMerge(estimator)) {
+ continue;
+ } else {
+ ndvEstimator = null;
+ break;
+ }
+ }
+ }
+ }
+ if (ndvEstimator != null) {
+ ndvEstimator = NumDistinctValueEstimatorFactory
+ .getEmptyNumDistinctValueEstimator(ndvEstimator);
+ }
+ LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ if (doAllPartitionContainStats || css.size() < 2) {
+ DecimalColumnStatsData aggregateData = null;
+ long lowerBound = 0;
+ long higherBound = 0;
+ double densityAvgSum = 0.0;
+ for (ColumnStatistics cs : css) {
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
+ lowerBound = Math.max(lowerBound, newData.getNumDVs());
+ higherBound += newData.getNumDVs();
+ densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils
+ .decimalToDouble(newData.getLowValue())) / newData.getNumDVs();
+ if (ndvEstimator != null) {
+ ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors()));
+ }
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ if (MetaStoreUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils
+ .decimalToDouble(newData.getLowValue())) {
+ aggregateData.setLowValue(aggregateData.getLowValue());
+ } else {
+ aggregateData.setLowValue(newData.getLowValue());
+ }
+ if (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils
+ .decimalToDouble(newData.getHighValue())) {
+ aggregateData.setHighValue(aggregateData.getHighValue());
+ } else {
+ aggregateData.setHighValue(newData.getHighValue());
+ }
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ }
+ }
+ if (ndvEstimator != null) {
+ // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
+ // use uniform distribution assumption because we can merge bitvectors
+ // to get a good estimation.
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ } else {
+ long estimation;
+ if (useDensityFunctionForNDVEstimation) {
+ // We have estimation, lowerbound and higherbound. We use estimation
+ // if it is between lowerbound and higherbound.
+ double densityAvg = densityAvgSum / partNames.size();
+ estimation = (long) ((MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils
+ .decimalToDouble(aggregateData.getLowValue())) / densityAvg);
+ if (estimation < lowerBound) {
+ estimation = lowerBound;
+ } else if (estimation > higherBound) {
+ estimation = higherBound;
+ }
+ } else {
+ estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
+ }
+ aggregateData.setNumDVs(estimation);
+ }
+ columnStatisticsData.setDecimalStats(aggregateData);
+ } else {
+ // we need extrapolation
+ LOG.debug("start extrapolation for " + colName);
+ Map<String, Integer> indexMap = new HashMap<String, Integer>();
+ for (int index = 0; index < partNames.size(); index++) {
+ indexMap.put(partNames.get(index), index);
+ }
+ Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
+ Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
+ // while we scan the css, we also get the densityAvg, lowerbound and
+ // higerbound when useDensityFunctionForNDVEstimation is true.
+ double densityAvgSum = 0.0;
+ if (ndvEstimator == null) {
+ // if not every partition uses bitvector for ndv, we just fall back to
+ // the traditional extrapolation methods.
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils
+ .decimalToDouble(newData.getLowValue())) / newData.getNumDVs();
+ }
+ adjustedIndexMap.put(partName, (double) indexMap.get(partName));
+ adjustedStatsMap.put(partName, cso.getStatsData());
+ }
+ } else {
+ // we first merge all the adjacent bitvectors that we could merge and
+ // derive new partition names and index.
+ StringBuilder pseudoPartName = new StringBuilder();
+ double pseudoIndexSum = 0;
+ int length = 0;
+ int curIndex = -1;
+ DecimalColumnStatsData aggregateData = null;
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
+ // newData.isSetBitVectors() should be true for sure because we
+ // already checked it before.
+ if (indexMap.get(partName) != curIndex) {
+ // There is bitvector, but it is not adjacent to the previous ones.
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setDecimalStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils
+ .decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs();
+ }
+ // reset everything
+ pseudoPartName = new StringBuilder();
+ pseudoIndexSum = 0;
+ length = 0;
+ ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
+ }
+ aggregateData = null;
+ }
+ curIndex = indexMap.get(partName);
+ pseudoPartName.append(partName);
+ pseudoIndexSum += curIndex;
+ length++;
+ curIndex++;
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ if (MetaStoreUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils
+ .decimalToDouble(newData.getLowValue())) {
+ aggregateData.setLowValue(aggregateData.getLowValue());
+ } else {
+ aggregateData.setLowValue(newData.getLowValue());
+ }
+ if (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils
+ .decimalToDouble(newData.getHighValue())) {
+ aggregateData.setHighValue(aggregateData.getHighValue());
+ } else {
+ aggregateData.setHighValue(newData.getHighValue());
+ }
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors()));
+ }
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setDecimalStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils
+ .decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs();
+ }
+ }
+ }
+ extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
+ adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
+ }
+ statsObj.setStatsData(columnStatisticsData);
+ LOG.debug("Ndv estimatation for " + colName + " is "
+ + columnStatisticsData.getDecimalStats().getNumDVs());
+ return statsObj;
+ }
+
+ @Override
+ public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+ int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+ Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
+ int rightBorderInd = numParts;
+ DecimalColumnStatsData extrapolateDecimalData = new DecimalColumnStatsData();
+ Map<String, DecimalColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
+ for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
+ extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDecimalStats());
+ }
+ List<Map.Entry<String, DecimalColumnStatsData>> list = new LinkedList<Map.Entry<String, DecimalColumnStatsData>>(
+ extractedAdjustedStatsMap.entrySet());
+ // get the lowValue
+ Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
+ public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
+ Map.Entry<String, DecimalColumnStatsData> o2) {
+ return o1.getValue().getLowValue().compareTo(o2.getValue().getLowValue());
+ }
+ });
+ double minInd = adjustedIndexMap.get(list.get(0).getKey());
+ double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ double lowValue = 0;
+ double min = MetaStoreUtils.decimalToDouble(list.get(0).getValue().getLowValue());
+ double max = MetaStoreUtils.decimalToDouble(list.get(list.size() - 1).getValue().getLowValue());
+ if (minInd == maxInd) {
+ lowValue = min;
+ } else if (minInd < maxInd) {
+ // left border is the min
+ lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
+ } else {
+ // right border is the min
+ lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ }
+
+ // get the highValue
+ Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
+ public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
+ Map.Entry<String, DecimalColumnStatsData> o2) {
+ return o1.getValue().getHighValue().compareTo(o2.getValue().getHighValue());
+ }
+ });
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ double highValue = 0;
+ min = MetaStoreUtils.decimalToDouble(list.get(0).getValue().getHighValue());
+ max = MetaStoreUtils.decimalToDouble(list.get(list.size() - 1).getValue().getHighValue());
+ if (minInd == maxInd) {
+ highValue = min;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ highValue = (min + (max - min) * minInd / (minInd - maxInd));
+ }
+
+ // get the #nulls
+ long numNulls = 0;
+ for (Map.Entry<String, DecimalColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
+ numNulls += entry.getValue().getNumNulls();
+ }
+ // we scale up sumNulls based on the number of partitions
+ numNulls = numNulls * numParts / numPartsWithStats;
+
+ // get the ndv
+ long ndv = 0;
+ long ndvMin = 0;
+ long ndvMax = 0;
+ Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
+ public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
+ Map.Entry<String, DecimalColumnStatsData> o2) {
+ return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
+ }
+ });
+ long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
+ long higherBound = 0;
+ for (Map.Entry<String, DecimalColumnStatsData> entry : list) {
+ higherBound += entry.getValue().getNumDVs();
+ }
+ if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
+ ndv = (long) ((highValue - lowValue) / densityAvg);
+ if (ndv < lowerBound) {
+ ndv = lowerBound;
+ } else if (ndv > higherBound) {
+ ndv = higherBound;
+ }
+ } else {
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ ndvMin = list.get(0).getValue().getNumDVs();
+ ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
+ if (minInd == maxInd) {
+ ndv = ndvMin;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
+ }
+ }
+ extrapolateDecimalData.setLowValue(StatObjectConverter.createThriftDecimal(String
+ .valueOf(lowValue)));
+ extrapolateDecimalData.setHighValue(StatObjectConverter.createThriftDecimal(String
+ .valueOf(highValue)));
+ extrapolateDecimalData.setNumNulls(numNulls);
+ extrapolateDecimalData.setNumDVs(ndv);
+ extrapolateData.setDecimalStats(extrapolateDecimalData);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java
new file mode 100644
index 0000000..e55c412
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java
@@ -0,0 +1,345 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implements
+ IExtrapolatePartStatus {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
+
+ @Override
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+
+ // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+ // bitvectors
+ boolean doAllPartitionContainStats = partNames.size() == css.size();
+ LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
+ NumDistinctValueEstimator ndvEstimator = null;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ if (!cso.getStatsData().getDoubleStats().isSetBitVectors()
+ || cso.getStatsData().getDoubleStats().getBitVectors().length() == 0) {
+ ndvEstimator = null;
+ break;
+ } else {
+ // check if all of the bit vectors can merge
+ NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(cso.getStatsData().getDoubleStats().getBitVectors());
+ if (ndvEstimator == null) {
+ ndvEstimator = estimator;
+ } else {
+ if (ndvEstimator.canMerge(estimator)) {
+ continue;
+ } else {
+ ndvEstimator = null;
+ break;
+ }
+ }
+ }
+ }
+ if (ndvEstimator != null) {
+ ndvEstimator = NumDistinctValueEstimatorFactory
+ .getEmptyNumDistinctValueEstimator(ndvEstimator);
+ }
+ LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ if (doAllPartitionContainStats || css.size() < 2) {
+ DoubleColumnStatsData aggregateData = null;
+ long lowerBound = 0;
+ long higherBound = 0;
+ double densityAvgSum = 0.0;
+ for (ColumnStatistics cs : css) {
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
+ lowerBound = Math.max(lowerBound, newData.getNumDVs());
+ higherBound += newData.getNumDVs();
+ densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+ if (ndvEstimator != null) {
+ ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors()));
+ }
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData
+ .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ }
+ }
+ if (ndvEstimator != null) {
+ // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
+ // use uniform distribution assumption because we can merge bitvectors
+ // to get a good estimation.
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ } else {
+ long estimation;
+ if (useDensityFunctionForNDVEstimation) {
+ // We have estimation, lowerbound and higherbound. We use estimation
+ // if it is between lowerbound and higherbound.
+ double densityAvg = densityAvgSum / partNames.size();
+ estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
+ if (estimation < lowerBound) {
+ estimation = lowerBound;
+ } else if (estimation > higherBound) {
+ estimation = higherBound;
+ }
+ } else {
+ estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
+ }
+ aggregateData.setNumDVs(estimation);
+ }
+ columnStatisticsData.setDoubleStats(aggregateData);
+ } else {
+ // we need extrapolation
+ LOG.debug("start extrapolation for " + colName);
+ Map<String, Integer> indexMap = new HashMap<String, Integer>();
+ for (int index = 0; index < partNames.size(); index++) {
+ indexMap.put(partNames.get(index), index);
+ }
+ Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
+ Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
+ // while we scan the css, we also get the densityAvg, lowerbound and
+ // higerbound when useDensityFunctionForNDVEstimation is true.
+ double densityAvgSum = 0.0;
+ if (ndvEstimator == null) {
+ // if not every partition uses bitvector for ndv, we just fall back to
+ // the traditional extrapolation methods.
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+ }
+ adjustedIndexMap.put(partName, (double) indexMap.get(partName));
+ adjustedStatsMap.put(partName, cso.getStatsData());
+ }
+ } else {
+ // we first merge all the adjacent bitvectors that we could merge and
+ // derive new partition names and index.
+ StringBuilder pseudoPartName = new StringBuilder();
+ double pseudoIndexSum = 0;
+ int length = 0;
+ int curIndex = -1;
+ DoubleColumnStatsData aggregateData = null;
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ DoubleColumnStatsData newData = cso.getStatsData().getDoubleStats();
+ // newData.isSetBitVectors() should be true for sure because we
+ // already checked it before.
+ if (indexMap.get(partName) != curIndex) {
+ // There is bitvector, but it is not adjacent to the previous ones.
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setDoubleStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+ }
+ // reset everything
+ pseudoPartName = new StringBuilder();
+ pseudoIndexSum = 0;
+ length = 0;
+ ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
+ }
+ aggregateData = null;
+ }
+ curIndex = indexMap.get(partName);
+ pseudoPartName.append(partName);
+ pseudoIndexSum += curIndex;
+ length++;
+ curIndex++;
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
+ newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors()));
+ }
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setDoubleStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+ }
+ }
+ }
+ extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
+ adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
+ }
+ LOG.debug("Ndv estimatation for " + colName + " is "
+ + columnStatisticsData.getDoubleStats().getNumDVs());
+ statsObj.setStatsData(columnStatisticsData);
+ return statsObj;
+ }
+
+ @Override
+ public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+ int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+ Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
+ int rightBorderInd = numParts;
+ DoubleColumnStatsData extrapolateDoubleData = new DoubleColumnStatsData();
+ Map<String, DoubleColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
+ for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
+ extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDoubleStats());
+ }
+ List<Map.Entry<String, DoubleColumnStatsData>> list = new LinkedList<Map.Entry<String, DoubleColumnStatsData>>(
+ extractedAdjustedStatsMap.entrySet());
+ // get the lowValue
+ Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
+ public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
+ Map.Entry<String, DoubleColumnStatsData> o2) {
+ return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
+ }
+ });
+ double minInd = adjustedIndexMap.get(list.get(0).getKey());
+ double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ double lowValue = 0;
+ double min = list.get(0).getValue().getLowValue();
+ double max = list.get(list.size() - 1).getValue().getLowValue();
+ if (minInd == maxInd) {
+ lowValue = min;
+ } else if (minInd < maxInd) {
+ // left border is the min
+ lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
+ } else {
+ // right border is the min
+ lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ }
+
+ // get the highValue
+ Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
+ public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
+ Map.Entry<String, DoubleColumnStatsData> o2) {
+ return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
+ }
+ });
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ double highValue = 0;
+ min = list.get(0).getValue().getHighValue();
+ max = list.get(list.size() - 1).getValue().getHighValue();
+ if (minInd == maxInd) {
+ highValue = min;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ highValue = (min + (max - min) * minInd / (minInd - maxInd));
+ }
+
+ // get the #nulls
+ long numNulls = 0;
+ for (Map.Entry<String, DoubleColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
+ numNulls += entry.getValue().getNumNulls();
+ }
+ // we scale up sumNulls based on the number of partitions
+ numNulls = numNulls * numParts / numPartsWithStats;
+
+ // get the ndv
+ long ndv = 0;
+ long ndvMin = 0;
+ long ndvMax = 0;
+ Collections.sort(list, new Comparator<Map.Entry<String, DoubleColumnStatsData>>() {
+ public int compare(Map.Entry<String, DoubleColumnStatsData> o1,
+ Map.Entry<String, DoubleColumnStatsData> o2) {
+ return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
+ }
+ });
+ long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
+ long higherBound = 0;
+ for (Map.Entry<String, DoubleColumnStatsData> entry : list) {
+ higherBound += entry.getValue().getNumDVs();
+ }
+ if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
+ ndv = (long) ((highValue - lowValue) / densityAvg);
+ if (ndv < lowerBound) {
+ ndv = lowerBound;
+ } else if (ndv > higherBound) {
+ ndv = higherBound;
+ }
+ } else {
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ ndvMin = list.get(0).getValue().getNumDVs();
+ ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
+ if (minInd == maxInd) {
+ ndv = ndvMin;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
+ }
+ }
+ extrapolateDoubleData.setLowValue(lowValue);
+ extrapolateDoubleData.setHighValue(highValue);
+ extrapolateDoubleData.setNumNulls(numNulls);
+ extrapolateDoubleData.setNumDVs(ndv);
+ extrapolateData.setDoubleStats(extrapolateDoubleData);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/IExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/IExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/IExtrapolatePartStatus.java
new file mode 100644
index 0000000..acf679e
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/IExtrapolatePartStatus.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+
+public interface IExtrapolatePartStatus {
+ // The following function will extrapolate the stats when the column stats of
+ // some partitions are missing.
+ /**
+ * @param extrapolateData
+ * it will carry back the specific stats, e.g., DOUBLE_STATS or
+ * LONG_STATS
+ * @param numParts
+ * the total number of partitions
+ * @param numPartsWithStats
+ * the number of partitions that have stats
+ * @param adjustedIndexMap
+ * the partition name to index map
+ * @param adjustedStatsMap
+ * the partition name to its stats map
+ * @param densityAvg
+ * the average of ndv density, which is useful when
+ * useDensityFunctionForNDVEstimation is true.
+ */
+ public abstract void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+ int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+ Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg);
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java
new file mode 100644
index 0000000..2ee09f3
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.aggr;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LongColumnStatsAggregator extends ColumnStatsAggregator implements
+ IExtrapolatePartStatus {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class);
+
+ @Override
+ public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
+ List<ColumnStatistics> css) throws MetaException {
+ ColumnStatisticsObj statsObj = null;
+
+ // check if all the ColumnStatisticsObjs contain stats and all the ndv are
+ // bitvectors
+ boolean doAllPartitionContainStats = partNames.size() == css.size();
+ LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
+ NumDistinctValueEstimator ndvEstimator = null;
+ String colType = null;
+ for (ColumnStatistics cs : css) {
+ if (cs.getStatsObjSize() != 1) {
+ throw new MetaException(
+ "The number of columns should be exactly one in aggrStats, but found "
+ + cs.getStatsObjSize());
+ }
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ if (statsObj == null) {
+ colType = cso.getColType();
+ statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
+ .getStatsData().getSetField());
+ }
+ if (!cso.getStatsData().getLongStats().isSetBitVectors()
+ || cso.getStatsData().getLongStats().getBitVectors().length() == 0) {
+ ndvEstimator = null;
+ break;
+ } else {
+ // check if all of the bit vectors can merge
+ NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(cso.getStatsData().getLongStats().getBitVectors());
+ if (ndvEstimator == null) {
+ ndvEstimator = estimator;
+ } else {
+ if (ndvEstimator.canMerge(estimator)) {
+ continue;
+ } else {
+ ndvEstimator = null;
+ break;
+ }
+ }
+ }
+ }
+ if (ndvEstimator != null) {
+ ndvEstimator = NumDistinctValueEstimatorFactory
+ .getEmptyNumDistinctValueEstimator(ndvEstimator);
+ }
+ LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
+ ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
+ if (doAllPartitionContainStats || css.size() < 2) {
+ LongColumnStatsData aggregateData = null;
+ long lowerBound = 0;
+ long higherBound = 0;
+ double densityAvgSum = 0.0;
+ for (ColumnStatistics cs : css) {
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ LongColumnStatsData newData = cso.getStatsData().getLongStats();
+ lowerBound = Math.max(lowerBound, newData.getNumDVs());
+ higherBound += newData.getNumDVs();
+ densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+ if (ndvEstimator != null) {
+ ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors()));
+ }
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData
+ .setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
+ }
+ }
+ if (ndvEstimator != null) {
+ // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
+ // use uniform distribution assumption because we can merge bitvectors
+ // to get a good estimation.
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ } else {
+ long estimation;
+ if (useDensityFunctionForNDVEstimation) {
+ // We have estimation, lowerbound and higherbound. We use estimation
+ // if it is between lowerbound and higherbound.
+ double densityAvg = densityAvgSum / partNames.size();
+ estimation = (long) ((aggregateData.getHighValue() - aggregateData.getLowValue()) / densityAvg);
+ if (estimation < lowerBound) {
+ estimation = lowerBound;
+ } else if (estimation > higherBound) {
+ estimation = higherBound;
+ }
+ } else {
+ estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
+ }
+ aggregateData.setNumDVs(estimation);
+ }
+ columnStatisticsData.setLongStats(aggregateData);
+ } else {
+ // we need extrapolation
+ LOG.debug("start extrapolation for " + colName);
+
+ Map<String, Integer> indexMap = new HashMap<String, Integer>();
+ for (int index = 0; index < partNames.size(); index++) {
+ indexMap.put(partNames.get(index), index);
+ }
+ Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
+ Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
+ // while we scan the css, we also get the densityAvg, lowerbound and
+ // higerbound when useDensityFunctionForNDVEstimation is true.
+ double densityAvgSum = 0.0;
+ if (ndvEstimator == null) {
+ // if not every partition uses bitvector for ndv, we just fall back to
+ // the traditional extrapolation methods.
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ LongColumnStatsData newData = cso.getStatsData().getLongStats();
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs();
+ }
+ adjustedIndexMap.put(partName, (double) indexMap.get(partName));
+ adjustedStatsMap.put(partName, cso.getStatsData());
+ }
+ } else {
+ // we first merge all the adjacent bitvectors that we could merge and
+ // derive new partition names and index.
+ StringBuilder pseudoPartName = new StringBuilder();
+ double pseudoIndexSum = 0;
+ int length = 0;
+ int curIndex = -1;
+ LongColumnStatsData aggregateData = null;
+ for (ColumnStatistics cs : css) {
+ String partName = cs.getStatsDesc().getPartName();
+ ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
+ LongColumnStatsData newData = cso.getStatsData().getLongStats();
+ // newData.isSetBitVectors() should be true for sure because we
+ // already checked it before.
+ if (indexMap.get(partName) != curIndex) {
+ // There is bitvector, but it is not adjacent to the previous ones.
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setLongStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+ }
+ // reset everything
+ pseudoPartName = new StringBuilder();
+ pseudoIndexSum = 0;
+ length = 0;
+ ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
+ }
+ aggregateData = null;
+ }
+ curIndex = indexMap.get(partName);
+ pseudoPartName.append(partName);
+ pseudoIndexSum += curIndex;
+ length++;
+ curIndex++;
+ if (aggregateData == null) {
+ aggregateData = newData.deepCopy();
+ } else {
+ aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
+ aggregateData.setHighValue(Math.max(aggregateData.getHighValue(),
+ newData.getHighValue()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+ ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
+ .getNumDistinctValueEstimator(newData.getBitVectors()));
+ }
+ if (length > 0) {
+ // we have to set ndv
+ adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
+ aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ csd.setLongStats(aggregateData);
+ adjustedStatsMap.put(pseudoPartName.toString(), csd);
+ if (useDensityFunctionForNDVEstimation) {
+ densityAvgSum += (aggregateData.getHighValue() - aggregateData.getLowValue()) / aggregateData.getNumDVs();
+ }
+ }
+ }
+ extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
+ adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
+ }
+ statsObj.setStatsData(columnStatisticsData);
+ LOG.debug("Ndv estimatation for " + colName + " is "
+ + columnStatisticsData.getLongStats().getNumDVs());
+ return statsObj;
+ }
+
+ @Override
+ public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
+ int numPartsWithStats, Map<String, Double> adjustedIndexMap,
+ Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
+ int rightBorderInd = numParts;
+ LongColumnStatsData extrapolateLongData = new LongColumnStatsData();
+ Map<String, LongColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
+ for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
+ extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getLongStats());
+ }
+ List<Map.Entry<String, LongColumnStatsData>> list = new LinkedList<Map.Entry<String, LongColumnStatsData>>(
+ extractedAdjustedStatsMap.entrySet());
+ // get the lowValue
+ Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
+ public int compare(Map.Entry<String, LongColumnStatsData> o1,
+ Map.Entry<String, LongColumnStatsData> o2) {
+ return o1.getValue().getLowValue() < o2.getValue().getLowValue() ? -1 : 1;
+ }
+ });
+ double minInd = adjustedIndexMap.get(list.get(0).getKey());
+ double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ long lowValue = 0;
+ long min = list.get(0).getValue().getLowValue();
+ long max = list.get(list.size() - 1).getValue().getLowValue();
+ if (minInd == maxInd) {
+ lowValue = min;
+ } else if (minInd < maxInd) {
+ // left border is the min
+ lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
+ } else {
+ // right border is the min
+ lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
+ }
+
+ // get the highValue
+ Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
+ public int compare(Map.Entry<String, LongColumnStatsData> o1,
+ Map.Entry<String, LongColumnStatsData> o2) {
+ return o1.getValue().getHighValue() < o2.getValue().getHighValue() ? -1 : 1;
+ }
+ });
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ long highValue = 0;
+ min = list.get(0).getValue().getHighValue();
+ max = list.get(list.size() - 1).getValue().getHighValue();
+ if (minInd == maxInd) {
+ highValue = min;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
+ }
+
+ // get the #nulls
+ long numNulls = 0;
+ for (Map.Entry<String, LongColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
+ numNulls += entry.getValue().getNumNulls();
+ }
+ // we scale up sumNulls based on the number of partitions
+ numNulls = numNulls * numParts / numPartsWithStats;
+
+ // get the ndv
+ long ndv = 0;
+ Collections.sort(list, new Comparator<Map.Entry<String, LongColumnStatsData>>() {
+ public int compare(Map.Entry<String, LongColumnStatsData> o1,
+ Map.Entry<String, LongColumnStatsData> o2) {
+ return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
+ }
+ });
+ long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
+ long higherBound = 0;
+ for (Map.Entry<String, LongColumnStatsData> entry : list) {
+ higherBound += entry.getValue().getNumDVs();
+ }
+ if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
+ ndv = (long) ((highValue - lowValue) / densityAvg);
+ if (ndv < lowerBound) {
+ ndv = lowerBound;
+ } else if (ndv > higherBound) {
+ ndv = higherBound;
+ }
+ } else {
+ minInd = adjustedIndexMap.get(list.get(0).getKey());
+ maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
+ min = list.get(0).getValue().getNumDVs();
+ max = list.get(list.size() - 1).getValue().getNumDVs();
+ if (minInd == maxInd) {
+ ndv = min;
+ } else if (minInd < maxInd) {
+ // right border is the max
+ ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
+ } else {
+ // left border is the max
+ ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
+ }
+ }
+ extrapolateLongData.setLowValue(lowValue);
+ extrapolateLongData.setHighValue(highValue);
+ extrapolateLongData.setNumNulls(numNulls);
+ extrapolateLongData.setNumDVs(ndv);
+ extrapolateData.setLongStats(extrapolateLongData);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
new file mode 100644
index 0000000..4c2d1bc
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+
+public class BinaryColumnStatsMerger extends ColumnStatsMerger {
+
+ @Override
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats();
+ BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats();
+ aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
+ aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
new file mode 100644
index 0000000..8e50153
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+
+public class BooleanColumnStatsMerger extends ColumnStatsMerger {
+
+ @Override
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ BooleanColumnStatsData aggregateData = aggregateColStats.getStatsData().getBooleanStats();
+ BooleanColumnStatsData newData = newColStats.getStatsData().getBooleanStats();
+ aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
+ aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
+ aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
new file mode 100644
index 0000000..474d4dd
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class ColumnStatsMerger {
+ protected final Logger LOG = LoggerFactory.getLogger(ColumnStatsMerger.class.getName());
+
+ public abstract void merge(ColumnStatisticsObj aggregateColStats,
+ ColumnStatisticsObj newColStats);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java
new file mode 100644
index 0000000..0ce1847
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.columnstats.merge;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
+import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+
+public class ColumnStatsMergerFactory {
+
+ private ColumnStatsMergerFactory() {
+ }
+
+ public static ColumnStatsMerger getColumnStatsMerger(ColumnStatisticsObj statsObjNew,
+ ColumnStatisticsObj statsObjOld) {
+ ColumnStatsMerger agg;
+ _Fields typeNew = statsObjNew.getStatsData().getSetField();
+ _Fields typeOld = statsObjOld.getStatsData().getSetField();
+ // make sure that they have the same type
+ typeNew = typeNew == typeOld ? typeNew : null;
+ switch (typeNew) {
+ case BOOLEAN_STATS:
+ agg = new BooleanColumnStatsMerger();
+ break;
+ case LONG_STATS: {
+ agg = new LongColumnStatsMerger();
+ break;
+ }
+ case DOUBLE_STATS: {
+ agg = new DoubleColumnStatsMerger();
+ break;
+ }
+ case STRING_STATS: {
+ agg = new StringColumnStatsMerger();
+ break;
+ }
+ case BINARY_STATS:
+ agg = new BinaryColumnStatsMerger();
+ break;
+ case DECIMAL_STATS: {
+ agg = new DecimalColumnStatsMerger();
+ break;
+ }
+ case DATE_STATS: {
+ agg = new DateColumnStatsMerger();
+ break;
+ }
+ default:
+ throw new IllegalArgumentException("Unknown stats type " + typeNew.toString());
+ }
+ return agg;
+ }
+
+ public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ ColumnStatisticsData csd = new ColumnStatisticsData();
+ cso.setColName(colName);
+ cso.setColType(colType);
+ switch (type) {
+ case BOOLEAN_STATS:
+ csd.setBooleanStats(new BooleanColumnStatsData());
+ break;
+
+ case LONG_STATS:
+ csd.setLongStats(new LongColumnStatsData());
+ break;
+
+ case DOUBLE_STATS:
+ csd.setDoubleStats(new DoubleColumnStatsData());
+ break;
+
+ case STRING_STATS:
+ csd.setStringStats(new StringColumnStatsData());
+ break;
+
+ case BINARY_STATS:
+ csd.setBinaryStats(new BinaryColumnStatsData());
+ break;
+
+ case DECIMAL_STATS:
+ csd.setDecimalStats(new DecimalColumnStatsData());
+ break;
+
+ case DATE_STATS:
+ csd.setDateStats(new DateColumnStatsData());
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unknown stats type");
+ }
+
+ cso.setStatsData(csd);
+ return cso;
+ }
+
+}
[16/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
deleted file mode 100644
index aefafe0..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
-import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
-import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
-import org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics;
-import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-
-/**
- * Test HMS Metrics on HBase Metastore
- */
-public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {
-
- @BeforeClass
- public static void startup() throws Exception {
- HBaseIntegrationTests.startMiniCluster();
- }
-
- @AfterClass
- public static void shutdown() throws Exception {
- HBaseIntegrationTests.shutdownMiniCluster();
- }
-
- @Before
- public void before() throws IOException {
- HBaseReadWrite.setConf(conf);
- conf = new HiveConf();
- conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
- "org.apache.hadoop.hive.metastore.hbase.HBaseStore");
- conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true);
- conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
-
- conf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true);
- conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
- SessionState.start(new CliSessionState(conf));
- driver = new Driver(conf);
- }
-
- @Test
- public void testMetaDataCounts() throws Exception {
- //1 databases created
- driver.run("create database testdb1");
-
- //4 tables
- driver.run("create table testtbl1 (key string)");
- driver.run("create table testtblpart (key string) partitioned by (partkey string)");
- driver.run("use testdb1");
- driver.run("create table testtbl2 (key string)");
- driver.run("create table testtblpart2 (key string) partitioned by (partkey string)");
-
- //6 partitions
- driver.run("alter table default.testtblpart add partition (partkey='a')");
- driver.run("alter table default.testtblpart add partition (partkey='b')");
- driver.run("alter table default.testtblpart add partition (partkey='c')");
- driver.run("alter table testdb1.testtblpart2 add partition (partkey='a')");
- driver.run("alter table testdb1.testtblpart2 add partition (partkey='b')");
- driver.run("alter table testdb1.testtblpart2 add partition (partkey='c')");
-
-
- //create and drop some additional metadata, to test drop counts.
- driver.run("create database tempdb");
- driver.run("use tempdb");
-
- driver.run("create table delete_by_table (key string) partitioned by (partkey string)");
- driver.run("alter table delete_by_table add partition (partkey='temp')");
- driver.run("drop table delete_by_table");
-
- driver.run("create table delete_by_part (key string) partitioned by (partkey string)");
- driver.run("alter table delete_by_part add partition (partkey='temp')");
- driver.run("alter table delete_by_part drop partition (partkey='temp')");
-
- driver.run("create table delete_by_db (key string) partitioned by (partkey string)");
- driver.run("alter table delete_by_db add partition (partkey='temp')");
- driver.run("use default");
- driver.run("drop database tempdb cascade");
-
- CodahaleMetrics metrics = (CodahaleMetrics) MetricsFactory.getInstance();
- String json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_DATABASES, 2);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_TABLES, 7);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_PARTITIONS, 9);
-
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.DELETE_TOTAL_DATABASES, 1);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.DELETE_TOTAL_TABLES, 3);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.DELETE_TOTAL_PARTITIONS, 3);
-
-
- //to test initial metadata count metrics.
- conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, ObjectStore.class.getName());
- HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("test", conf, false);
- baseHandler.init();
- baseHandler.updateMetrics();
-
- //1 new db + default
- json = metrics.dumpJson();
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_DATABASES, 2);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_TABLES, 4);
- MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_PARTITIONS, 6);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
deleted file mode 100644
index d4966b9..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-
-/**
- * Integration tests with HBase Mini-cluster using actual SQL
- */
-public class TestHBaseMetastoreSql extends HBaseIntegrationTests {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreIntegration.class.getName());
-
- @BeforeClass
- public static void startup() throws Exception {
- HBaseIntegrationTests.startMiniCluster();
-
- }
-
- @AfterClass
- public static void shutdown() throws Exception {
- HBaseIntegrationTests.shutdownMiniCluster();
- }
-
- @Before
- public void before() throws IOException {
- setupConnection();
- setupDriver();
- }
-
- @Test
- public void insertIntoTable() throws Exception {
- driver.run("create table iit (c int)");
- CommandProcessorResponse rsp = driver.run("insert into table iit values (3)");
- Assert.assertEquals(0, rsp.getResponseCode());
- }
-
- @Test
- public void insertIntoPartitionTable() throws Exception {
- driver.run("create table iipt (c int) partitioned by (ds string)");
- CommandProcessorResponse rsp =
- driver.run("insert into table iipt partition(ds) values (1, 'today'), (2, 'yesterday')," +
- "(3, 'tomorrow')");
- Assert.assertEquals(0, rsp.getResponseCode());
- }
-
- @Test
- public void database() throws Exception {
- CommandProcessorResponse rsp = driver.run("create database db");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("set role admin");
- Assert.assertEquals(0, rsp.getResponseCode());
- // security doesn't let me change the properties
- rsp = driver.run("alter database db set dbproperties ('key' = 'value')");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("drop database db");
- Assert.assertEquals(0, rsp.getResponseCode());
- }
-
- @Test
- public void table() throws Exception {
- driver.run("create table tbl (c int)");
- CommandProcessorResponse rsp = driver.run("insert into table tbl values (3)");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("select * from tbl");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("alter table tbl set tblproperties ('example', 'true')");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("drop table tbl");
- Assert.assertEquals(0, rsp.getResponseCode());
- }
-
- @Test
- public void partitionedTable() throws Exception {
- driver.run("create table parttbl (c int) partitioned by (ds string)");
- CommandProcessorResponse rsp =
- driver.run("insert into table parttbl partition(ds) values (1, 'today'), (2, 'yesterday')" +
- ", (3, 'tomorrow')");
- Assert.assertEquals(0, rsp.getResponseCode());
- // Do it again, to check insert into existing partitions
- rsp = driver.run("insert into table parttbl partition(ds) values (4, 'today'), (5, 'yesterday')"
- + ", (6, 'tomorrow')");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("insert into table parttbl partition(ds = 'someday') values (1)");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("insert into table parttbl partition(ds = 'someday') values (2)");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("alter table parttbl add partition (ds = 'whenever')");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("insert into table parttbl partition(ds = 'whenever') values (2)");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("alter table parttbl touch partition (ds = 'whenever')");
- Assert.assertEquals(0, rsp.getResponseCode());
- // TODO - Can't do this until getPartitionsByExpr implemented
- /*
- rsp = driver.run("alter table parttbl drop partition (ds = 'whenever')");
- Assert.assertEquals(0, rsp.getResponseCode());
- */
- rsp = driver.run("select * from parttbl");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("select * from parttbl where ds = 'today'");
- Assert.assertEquals(0, rsp.getResponseCode());
- }
-
- @Test
- public void role() throws Exception {
- CommandProcessorResponse rsp = driver.run("set role admin");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("create role role1");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("grant role1 to user fred with admin option");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("create role role2");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("grant role1 to role role2");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("show principals role1");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("show role grant role role1");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("show role grant user " + System.getProperty("user.name"));
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("show roles");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("revoke admin option for role1 from user fred");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("revoke role1 from user fred");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("revoke role1 from role role2");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("show current roles");
- Assert.assertEquals(0, rsp.getResponseCode());
-
- rsp = driver.run("drop role role2");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("drop role role1");
- Assert.assertEquals(0, rsp.getResponseCode());
- }
-
- @Test
- public void grant() throws Exception {
- CommandProcessorResponse rsp = driver.run("set role admin");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("create role role3");
- Assert.assertEquals(0, rsp.getResponseCode());
- driver.run("create table granttbl (c int)");
- Assert.assertEquals(0, rsp.getResponseCode());
- driver.run("grant select on granttbl to " + System.getProperty("user.name"));
- Assert.assertEquals(0, rsp.getResponseCode());
- driver.run("grant select on granttbl to role3 with grant option");
- Assert.assertEquals(0, rsp.getResponseCode());
- driver.run("revoke select on granttbl from " + System.getProperty("user.name"));
- Assert.assertEquals(0, rsp.getResponseCode());
- driver.run("revoke grant option for select on granttbl from role3");
- Assert.assertEquals(0, rsp.getResponseCode());
- }
-
- @Test
- public void describeNonpartitionedTable() throws Exception {
- CommandProcessorResponse rsp = driver.run("create table alter1(a int, b int)");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("describe extended alter1");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("alter table alter1 set serdeproperties('s1'='9')");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("describe extended alter1");
- Assert.assertEquals(0, rsp.getResponseCode());
- }
-
- @Test
- public void alterRenamePartitioned() throws Exception {
- driver.run("create table alterrename (c int) partitioned by (ds string)");
- driver.run("alter table alterrename add partition (ds = 'a')");
- CommandProcessorResponse rsp = driver.run("describe extended alterrename partition (ds='a')");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("alter table alterrename rename to alter_renamed");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("describe extended alter_renamed partition (ds='a')");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("describe extended alterrename partition (ds='a')");
- Assert.assertEquals(10001, rsp.getResponseCode());
- }
-
- @Test
- public void alterRename() throws Exception {
- driver.run("create table alterrename1 (c int)");
- CommandProcessorResponse rsp = driver.run("describe alterrename1");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("alter table alterrename1 rename to alter_renamed1");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("describe alter_renamed1");
- Assert.assertEquals(0, rsp.getResponseCode());
- rsp = driver.run("describe alterrename1");
- Assert.assertEquals(10001, rsp.getResponseCode());
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java
deleted file mode 100644
index c98911a..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool.java
+++ /dev/null
@@ -1,582 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.HiveObjectType;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-public class TestHBaseSchemaTool extends HBaseIntegrationTests {
-
- private final String lsep = System.getProperty("line.separator");
-
- @BeforeClass
- public static void startup() throws Exception {
- HBaseIntegrationTests.startMiniCluster();
- }
-
- @AfterClass
- public static void shutdown() throws Exception {
- HBaseIntegrationTests.shutdownMiniCluster();
- }
-
- @Before
- public void setup() throws IOException {
- setupHBaseStore();
- }
-
- @Test
- public void listTables() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(true, null, null, null, conf, out, err);
- Assert.assertEquals(StringUtils.join(HBaseReadWrite.tableNames, lsep) + lsep,
- outStr.toString());
- }
-
- @Test
- public void bogusTable() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, "nosuch", null, null, conf, out, err);
- Assert.assertEquals("Unknown table: nosuch" + lsep, errStr.toString());
- }
-
- @Test
- public void noSuchDb() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.DB_TABLE, "nosuch", null, conf, out, err);
- Assert.assertEquals("No such database: nosuch" + lsep, outStr.toString());
- }
-
- @Test
- public void noMatchingDb() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.DB_TABLE, null, "nomatch", conf, out, err);
- Assert.assertEquals("No matching database: nomatch" + lsep, outStr.toString());
- }
-
- @Test
- public void noSuchRole() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.ROLE_TABLE, "nosuch", null, conf, out, err);
- Assert.assertEquals("No such role: nosuch" + lsep, outStr.toString());
- }
-
- @Test
- public void noMatchingRole() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.ROLE_TABLE, null, "nomatch", conf, out, err);
- Assert.assertEquals("No matching role: nomatch" + lsep, outStr.toString());
- }
-
- @Test
- public void noSuchUser() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, "nosuch", null, conf, out, err);
- Assert.assertEquals("No such user: nosuch" + lsep, outStr.toString());
- }
-
- @Test
- public void noMatchingUser() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, null, "nomatch", conf, out, err);
- Assert.assertEquals("No matching user: nomatch" + lsep, outStr.toString());
- }
-
- @Test
- public void noSuchFunction() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.FUNC_TABLE, "nosuch", null, conf, out, err);
- Assert.assertEquals("No such function: nosuch" + lsep, outStr.toString());
- }
-
- @Test
- public void noMatchingFunction() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.FUNC_TABLE, null, "nomatch", conf, out,
- err);
- Assert.assertEquals("No matching function: nomatch" + lsep, outStr.toString());
- }
-
- @Test
- public void noSuchTable() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.TABLE_TABLE, "nosuch", null, conf, out, err);
- Assert.assertEquals("No such table: nosuch" + lsep, outStr.toString());
- }
-
- @Test
- public void noMatchingTable() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.TABLE_TABLE, null, "nomatch", conf, out, err);
- Assert.assertEquals("No matching table: nomatch" + lsep, outStr.toString());
- }
-
- @Test
- public void noSuchPart() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, "nosuch", null, conf, out, err);
- Assert.assertEquals("No such partition: nosuch" + lsep, outStr.toString());
- }
-
- @Test
- public void noSuchPartValidFormat() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
- // Test with something that looks like a valid entry
- new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, "default.nosuch.nosuch", null, conf,
- out, err);
- Assert.assertEquals("No such partition: default.nosuch.nosuch" + lsep, outStr.toString());
- }
-
-
- @Test
- public void noMatchingPart() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, null, "nomatch", conf, out, err);
- Assert.assertEquals("No matching partition: nomatch" + lsep, outStr.toString());
- }
-
- @Test
- public void noMatchingPartValidFormat() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- new HBaseSchemaTool().go(false, HBaseReadWrite.PART_TABLE, null, "nomatch.a.b", conf, out, err);
- Assert.assertEquals("No matching partition: nomatch.a.b" + lsep, outStr.toString());
- }
-
- @Test
- public void noSuchStorageDescriptor() throws Exception {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- // Strangly enough things don't come back quite the same when going through the Base64
- // encode/decode.
- new HBaseSchemaTool().go(false, HBaseReadWrite.SD_TABLE, "nosuch", null, conf, out, err);
- Assert.assertEquals("No such storage descriptor: nosucg" + lsep, outStr.toString());
- }
-
- @Test
- public void oneMondoTest() throws Exception {
- // This is a pain to do in one big test, but we have to control the order so that we have tests
- // without dbs, etc.
- HBaseSchemaTool tool = new HBaseSchemaTool();
-
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- // This needs to be up front before we create any tables or partitions
- tool.go(false, HBaseReadWrite.SD_TABLE, null, "whatever", conf, out, err);
- Assert.assertEquals("No storage descriptors" + lsep, outStr.toString());
-
- // This one needs to be up front too
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.SEQUENCES_TABLE, null, "whatever", conf, out, err);
- Assert.assertEquals("No sequences" + lsep, outStr.toString());
-
- // Create some databases
- String[] dbNames = new String[3];
- for (int i = 0; i < dbNames.length; i++) {
- dbNames[i] = "db" + i;
- Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
- }
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.DB_TABLE, "db0", null, conf, out, err);
- Assert.assertEquals("{\"name\":\"db0\",\"description\":\"no description\"," +
- "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep, outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.DB_TABLE, null, ".*", conf, out, err);
- Assert.assertEquals("{\"name\":\"db0\",\"description\":\"no description\"," +
- "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep +
- "{\"name\":\"db1\",\"description\":\"no description\"," +
- "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep +
- "{\"name\":\"db2\",\"description\":\"no description\"," +
- "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep,
- outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.DB_TABLE, null, "db[12]", conf, out, err);
- Assert.assertEquals("{\"name\":\"db1\",\"description\":\"no description\"," +
- "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep +
- "{\"name\":\"db2\",\"description\":\"no description\"," +
- "\"locationUri\":\"file:///tmp\",\"parameters\":{}}" + lsep,
- outStr.toString());
-
- String[] roleNames = new String[2];
- for (int i = 0; i < roleNames.length; i++) {
- roleNames[i] = "role" + i;
- store.addRole(roleNames[i], "me");
- }
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.ROLE_TABLE, null, "role.", conf, out, err);
- Assert.assertEquals("{\"roleName\":\"role0\",\"createTime\":now,\"ownerName\":\"me\"}" +
- lsep + "{\"roleName\":\"role1\",\"createTime\":now,\"ownerName\":\"me\"}" + lsep,
- outStr.toString().replaceAll("createTime\":[0-9]+", "createTime\":now"));
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.ROLE_TABLE, "role1", null, conf, out, err);
- Assert.assertEquals("{\"roleName\":\"role1\",\"createTime\":now,\"ownerName\":\"me\"}" + lsep,
- outStr.toString().replaceAll("createTime\":[0-9]+", "createTime\":now"));
-
- Role role1 = store.getRole("role1");
- store.grantRole(role1, "fred", PrincipalType.USER, "me", PrincipalType.USER, false);
- store.grantRole(role1, "joanne", PrincipalType.USER, "me", PrincipalType.USER, false);
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, null, ".*", conf, out, err);
- Assert.assertEquals("fred: role1" + lsep + "joanne: role1" + lsep, outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.USER_TO_ROLE_TABLE, "joanne", null, conf, out, err);
- Assert.assertEquals("role1" + lsep, outStr.toString());
-
- String[] funcNames = new String[3];
- for (int i = 0; i < funcNames.length; i++) {
- funcNames[i] = "func" + i;
- Function function = new Function(funcNames[i], "db1", "Function", "me", PrincipalType.USER, 0,
- FunctionType.JAVA, null);
- store.createFunction(function);
- }
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.FUNC_TABLE, "db1.func0", null, conf, out, err);
- Assert.assertEquals("{\"functionName\":\"func0\",\"dbName\":\"db1\"," +
- "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
- "\"functionType\":1}" + lsep, outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.FUNC_TABLE, null, ".*", conf, out, err);
- Assert.assertEquals("{\"functionName\":\"func0\",\"dbName\":\"db1\"," +
- "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
- "\"functionType\":1}" + lsep +
- "{\"functionName\":\"func1\",\"dbName\":\"db1\"," +
- "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
- "\"functionType\":1}" + lsep +
- "{\"functionName\":\"func2\",\"dbName\":\"db1\"," +
- "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
- "\"functionType\":1}" + lsep, outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.FUNC_TABLE, null, "db1.func[12]", conf, out, err);
- Assert.assertEquals("{\"functionName\":\"func1\",\"dbName\":\"db1\"," +
- "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
- "\"functionType\":1}" + lsep +
- "{\"functionName\":\"func2\",\"dbName\":\"db1\"," +
- "\"className\":\"Function\",\"ownerName\":\"me\",\"ownerType\":1,\"createTime\":0," +
- "\"functionType\":1}" + lsep, outStr.toString());
-
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.GLOBAL_PRIVS_TABLE, null, null, conf, out, err);
- Assert.assertEquals("No global privileges" + lsep, outStr.toString());
-
- List<HiveObjectPrivilege> privileges = new ArrayList<>();
- HiveObjectRef hiveObjRef = new HiveObjectRef(HiveObjectType.GLOBAL, "db0", "tab0", null,
- null);
- PrivilegeGrantInfo grantInfo =
- new PrivilegeGrantInfo("read", 0, "me", PrincipalType.USER, false);
- HiveObjectPrivilege hop = new HiveObjectPrivilege(hiveObjRef, "user", PrincipalType.USER,
- grantInfo);
- privileges.add(hop);
-
- grantInfo = new PrivilegeGrantInfo("create", 0, "me", PrincipalType.USER, true);
- hop = new HiveObjectPrivilege(hiveObjRef, "user", PrincipalType.USER, grantInfo);
- privileges.add(hop);
-
- PrivilegeBag pBag = new PrivilegeBag(privileges);
- store.grantPrivileges(pBag);
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.GLOBAL_PRIVS_TABLE, null, null, conf, out, err);
- Assert.assertEquals(
- "{\"userPrivileges\":{\"user\":[{\"privilege\":\"read\",\"createTime\":0," +
- "\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":0},{\"privilege\":\"create\"," +
- "\"createTime\":0,\"grantor\":\"me\",\"grantorType\":1,\"grantOption\":1}]}}" + lsep,
- outStr.toString());
-
-
- String[] tableNames = new String[3];
- for (int i = 0; i < tableNames.length; i++) {
- tableNames[i] = "tab" + i;
- StorageDescriptor sd = new StorageDescriptor(Arrays.asList(new FieldSchema("col1", "int",
- ""), new FieldSchema("col2", "varchar(32)", "")),
- "/tmp", null, null, false, 0, null, null, null, Collections.<String, String>emptyMap());
- Table tab = new Table(tableNames[i], dbNames[0], "me", 0, 0, 0, sd,
- Arrays.asList(new FieldSchema("pcol1", "string", ""),
- new FieldSchema("pcol2", "string", "")),
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(tab);
- }
-
- ColumnStatisticsDesc tableStatsDesc = new ColumnStatisticsDesc(false, "db0", "tab0");
- ColumnStatisticsData tcsd = new ColumnStatisticsData();
- LongColumnStatsData tlcsd = new LongColumnStatsData(1, 2);
- tlcsd.setLowValue(-95);
- tlcsd.setHighValue(95);
- tcsd.setLongStats(tlcsd);
- ColumnStatisticsData tcsd2 = new ColumnStatisticsData();
- tcsd2.setStringStats(new StringColumnStatsData(97, 18.78, 29, 397));
- List<ColumnStatisticsObj> tcsos = Arrays.asList(
- new ColumnStatisticsObj("col1", "int", tcsd),
- new ColumnStatisticsObj("col2", "varchar(32)", tcsd2));
- ColumnStatistics tStatObj = new ColumnStatistics(tableStatsDesc, tcsos);
- store.updateTableColumnStatistics(tStatObj);
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.TABLE_TABLE, "db0.tab1", null, conf, out, err);
- Assert.assertEquals("{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\"," +
- "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," +
- "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"}," +
- "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{}," +
- "\"tableType\":\"\",\"rewriteEnabled\":0} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep,
- outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.TABLE_TABLE, null, "db0.*", conf, out, err);
- Assert.assertEquals("{\"tableName\":\"tab0\",\"dbName\":\"db0\",\"owner\":\"me\"," +
- "\"createTime\":0,\"lastAccessTime\":0,\"retention\":0," +
- "\"partitionKeys\":[{\"name\":\"pcol1\",\"type\":\"string\",\"comment\":\"\"}," +
- "{\"name\":\"pcol2\",\"type\":\"string\",\"comment\":\"\"}],\"parameters\":{\"COLUMN_STATS_ACCURATE\":\"{\\\"COLUMN_STATS\\\":{\\\"col1\\\":\\\"true\\\",\\\"col2\\\":\\\"true\\\"}}\"}," +
- "\"tableType\":\"\",\"rewriteEnabled\":0} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column " +
- "col1: {\"colName\":\"col1\",\"colType\":\"int\"," +
- "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," +
- "\"numDVs\":2,\"bitVectors\":\"\"}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," +
- "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78," +
- "\"numNulls\":29,\"numDVs\":397,\"bitVectors\":\"\"}}}" + lsep +
- "{\"tableName\":\"tab1\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0," +
- "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\"," +
- "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\"," +
- "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\",\"rewriteEnabled\":0} sdHash: " +
- "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep +
- "{\"tableName\":\"tab2\",\"dbName\":\"db0\",\"owner\":\"me\",\"createTime\":0," +
- "\"lastAccessTime\":0,\"retention\":0,\"partitionKeys\":[{\"name\":\"pcol1\"," +
- "\"type\":\"string\",\"comment\":\"\"},{\"name\":\"pcol2\",\"type\":\"string\"," +
- "\"comment\":\"\"}],\"parameters\":{},\"tableType\":\"\",\"rewriteEnabled\":0} sdHash: " +
- "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep, outStr.toString());
-
- List<List<String>> partVals = Arrays.asList(Arrays.asList("a", "b"), Arrays.asList("c", "d"));
- for (List<String> pv : partVals) {
- StorageDescriptor sd = new StorageDescriptor(Arrays.asList(new FieldSchema("col1", "int",
- ""), new FieldSchema("col2", "varchar(32)", "")),
- "/tmp", null, null, false, 0, null, null, null, Collections.<String, String>emptyMap());
- Partition p = new Partition(pv, "db0", "tab1", 0, 0, sd, Collections.<String, String>emptyMap());
- store.addPartition(p);
- }
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.PART_TABLE, "db0.tab1.a.b", null, conf, out, err);
- Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
- "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: " +
- "qQTgZAi5VzgpozzFGmIVTQ stats:" + lsep, outStr.toString());
-
- ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "db0", "tab1");
- statsDesc.setPartName("pcol1=c/pcol2=d");
- ColumnStatisticsData csd1 = new ColumnStatisticsData();
- LongColumnStatsData lcsd = new LongColumnStatsData(1, 2);
- lcsd.setLowValue(-95);
- lcsd.setHighValue(95);
- csd1.setLongStats(lcsd);
- ColumnStatisticsData csd2 = new ColumnStatisticsData();
- csd2.setStringStats(new StringColumnStatsData(97, 18.78, 29, 397));
- List<ColumnStatisticsObj> csos = Arrays.asList(
- new ColumnStatisticsObj("col1", "int", csd1),
- new ColumnStatisticsObj("col2", "varchar(32)", csd2));
- ColumnStatistics statsObj = new ColumnStatistics(statsDesc, csos);
- store.updatePartitionColumnStatistics(statsObj, partVals.get(1));
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.PART_TABLE, "db0.tab1.c.d", null, conf, out, err);
- Assert.assertEquals("{\"values\":[\"c\",\"d\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
- "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{\"COLUMN_STATS_ACCURATE\":\"{\\\"COLUMN_STATS\\\":{\\\"col1\\\":\\\"true\\\",\\\"col2\\\":\\\"true\\\"}}\"}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " +
- "stats: column col1: {\"colName\":\"col1\",\"colType\":\"int\"," +
- "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," +
- "\"numDVs\":2,\"bitVectors\":\"\"}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," +
- "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29," +
- "\"numDVs\":397,\"bitVectors\":\"\"}}}" + lsep, outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.PART_TABLE, null, "db0.tab1", conf, out, err);
- Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
- "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " +
- "stats:" + lsep +
- "{\"values\":[\"c\",\"d\"],\"dbName\":\"db0\",\"tableName\":\"tab1\",\"createTime\":0," +
- "\"lastAccessTime\":0,\"parameters\":{\"COLUMN_STATS_ACCURATE\":\"{\\\"COLUMN_STATS\\\":{\\\"col1\\\":\\\"true\\\",\\\"col2\\\":\\\"true\\\"}}\"}} sdHash: qQTgZAi5VzgpozzFGmIVTQ stats: column " +
- "col1: {\"colName\":\"col1\",\"colType\":\"int\"," +
- "\"statsData\":{\"longStats\":{\"lowValue\":-95,\"highValue\":95,\"numNulls\":1," +
- "\"numDVs\":2,\"bitVectors\":\"\"}}} column col2: {\"colName\":\"col2\",\"colType\":\"varchar(32)\"," +
- "\"statsData\":{\"stringStats\":{\"maxColLen\":97,\"avgColLen\":18.78,\"numNulls\":29," +
- "\"numDVs\":397,\"bitVectors\":\"\"}}}" + lsep, outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.PART_TABLE, null, "db0.tab1.a", conf, out, err);
- Assert.assertEquals("{\"values\":[\"a\",\"b\"],\"dbName\":\"db0\",\"tableName\":\"tab1\"," +
- "\"createTime\":0,\"lastAccessTime\":0,\"parameters\":{}} sdHash: qQTgZAi5VzgpozzFGmIVTQ " +
- "stats:" + lsep, outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.SD_TABLE, "qQTgZAi5VzgpozzFGmIVTQ", null, conf, out, err);
- Assert.assertEquals("{\"cols\":[{\"name\":\"col1\",\"type\":\"int\",\"comment\":\"\"}," +
- "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0," +
- "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}" + lsep,
- outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.SD_TABLE, null, "whatever", conf, out, err);
- Assert.assertEquals("qQTgZAi5VzgpozzFGmIVTQ: {\"cols\":[{\"name\":\"col1\",\"type\":\"int\"," +
- "\"comment\":\"\"}," +
- "{\"name\":\"col2\",\"type\":\"varchar(32)\",\"comment\":\"\"}],\"compressed\":0," +
- "\"numBuckets\":0,\"bucketCols\":[],\"sortCols\":[],\"storedAsSubDirectories\":0}" + lsep,
- outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.SECURITY_TABLE, null, "whatever", conf, out, err);
- Assert.assertEquals("No security related entries" + lsep, outStr.toString());
-
- store.addMasterKey("this be a key");
- store.addToken("tokenid", "delegation token");
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.SECURITY_TABLE, null, "whatever", conf, out, err);
- Assert.assertEquals("Master key 0: this be a key" + lsep +
- "Delegation token tokenid: delegation token" + lsep, outStr.toString());
-
- outStr = new ByteArrayOutputStream();
- out = new PrintStream(outStr);
- tool.go(false, HBaseReadWrite.SEQUENCES_TABLE, null, "whatever", conf, out, err);
- Assert.assertEquals("master_key: 1" + lsep, outStr.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java
deleted file mode 100644
index 0c95b2f..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseSchemaTool2.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.commons.lang.StringUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-
-/**
- * This is in a separate class because install tests shouldn't set up the metastore first.
- */
-public class TestHBaseSchemaTool2 extends HBaseIntegrationTests {
-
- private String lsep = System.getProperty("line.separator");
-
- @BeforeClass
- public static void startup() throws Exception {
- HBaseIntegrationTests.startMiniCluster();
- }
-
- @AfterClass
- public static void shutdown() throws Exception {
- HBaseIntegrationTests.shutdownMiniCluster();
- }
-
- @Test
- public void install() {
- ByteArrayOutputStream outStr = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(outStr);
- ByteArrayOutputStream errStr = new ByteArrayOutputStream();
- PrintStream err = new PrintStream(errStr);
-
- HBaseSchemaTool tool = new HBaseSchemaTool();
- tool.install(conf, err);
- tool.go(true, null, null, null, conf, out, err);
- Assert.assertEquals(StringUtils.join(HBaseReadWrite.tableNames, lsep) + lsep,
- outStr.toString());
- }
-}
[03/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java
deleted file mode 100644
index 06884b3..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java
+++ /dev/null
@@ -1,483 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hive.metastore.PartFilterExprUtil;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.FilterPlan;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.MultiScanPlan;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan.ScanMarker;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.primitives.Shorts;
-
-public class TestHBaseFilterPlanUtil {
- final boolean INCLUSIVE = true;
-
- /**
- * Test the function that compares byte arrays
- */
- @Test
- public void testCompare() {
-
- Assert.assertEquals(-1, HBaseFilterPlanUtil.compare(new byte[] { 1, 2 }, new byte[] { 1, 3 }));
- Assert.assertEquals(-1,
- HBaseFilterPlanUtil.compare(new byte[] { 1, 2, 3 }, new byte[] { 1, 3 }));
- Assert.assertEquals(-1,
- HBaseFilterPlanUtil.compare(new byte[] { 1, 2 }, new byte[] { 1, 2, 3 }));
-
- Assert.assertEquals(0, HBaseFilterPlanUtil.compare(new byte[] { 3, 2 }, new byte[] { 3, 2 }));
-
- Assert
- .assertEquals(1, HBaseFilterPlanUtil.compare(new byte[] { 3, 2, 1 }, new byte[] { 3, 2 }));
- Assert
- .assertEquals(1, HBaseFilterPlanUtil.compare(new byte[] { 3, 3, 1 }, new byte[] { 3, 2 }));
-
- }
-
- /**
- * Test function that finds greater/lesser marker
- */
- @Test
- public void testgetComparedMarker() {
- ScanMarker l;
- ScanMarker r;
-
- // equal plans
- l = new ScanMarker("1", INCLUSIVE, "int");
- r = new ScanMarker("1", INCLUSIVE, "int");
- assertFirstGreater(l, r);
-
- l = new ScanMarker("1", !INCLUSIVE, "int");
- r = new ScanMarker("1", !INCLUSIVE, "int");
- assertFirstGreater(l, r);
-
- assertFirstGreater(null, null);
-
- // create l is greater because of inclusive flag
- l = new ScanMarker("1", !INCLUSIVE, "int");
- // the rule for null vs non-null is different
- // non-null is both smaller and greater than null
- Assert.assertEquals(l, ScanPlan.getComparedMarker(l, null, true));
- Assert.assertEquals(l, ScanPlan.getComparedMarker(null, l, true));
- Assert.assertEquals(l, ScanPlan.getComparedMarker(l, null, false));
- Assert.assertEquals(l, ScanPlan.getComparedMarker(null, l, false));
-
- // create l that is greater because of the bytes
- l = new ScanMarker("2", INCLUSIVE, "int");
- r = new ScanMarker("1", INCLUSIVE, "int");
- assertFirstGreater(l, r);
-
- }
-
- private void assertFirstGreater(ScanMarker big, ScanMarker small) {
- Assert.assertEquals(big, ScanPlan.getComparedMarker(big, small, true));
- Assert.assertEquals(big, ScanPlan.getComparedMarker(small, big, true));
- Assert.assertEquals(small, ScanPlan.getComparedMarker(big, small, false));
- Assert.assertEquals(small, ScanPlan.getComparedMarker(small, big, false));
- }
-
- /**
- * Test ScanPlan AND operation
- */
- @Test
- public void testScanPlanAnd() {
- ScanPlan l = new ScanPlan();
- ScanPlan r = new ScanPlan();
- l.setStartMarker("a", "int", "10", INCLUSIVE);
- r.setStartMarker("a", "int", "10", INCLUSIVE);
-
- ScanPlan res;
- // both equal
- res = l.and(r).getPlans().get(0);
- Assert.assertEquals(new ScanMarker("10", INCLUSIVE, "int"), res.markers.get("a").startMarker);
-
- // add equal end markers as well, and test AND again
- l.setEndMarker("a", "int", "20", INCLUSIVE);
- r.setEndMarker("a", "int", "20", INCLUSIVE);
- res = l.and(r).getPlans().get(0);
- Assert.assertEquals(new ScanMarker("10", INCLUSIVE, "int"), res.markers.get("a").startMarker);
- Assert.assertEquals(new ScanMarker("20", INCLUSIVE, "int"), res.markers.get("a").endMarker);
-
- l.setStartMarker("a", "int", "10", !INCLUSIVE);
- l.setEndMarker("a", "int", "20", INCLUSIVE);
-
- r.setStartMarker("a", "int", "10", INCLUSIVE);
- r.setEndMarker("a", "int", "15", INCLUSIVE);
- res = l.and(r).getPlans().get(0);
- // start of l is greater, end of r is smaller
- Assert.assertEquals(l.markers.get("a").startMarker, res.markers.get("a").startMarker);
- Assert.assertEquals(r.markers.get("a").endMarker, res.markers.get("a").endMarker);
-
- }
-
- /**
- * Test ScanPlan OR operation
- */
- @Test
- public void testScanPlanOr() {
- ScanPlan l = new ScanPlan();
- ScanPlan r = new ScanPlan();
- l.setStartMarker("a", "int", "1", INCLUSIVE);
- r.setStartMarker("a", "int", "11", INCLUSIVE);
-
- FilterPlan res1 = l.or(r);
- Assert.assertEquals(2, res1.getPlans().size());
- res1.getPlans().get(0).markers.get("a").startMarker.equals(l.markers.get("a").startMarker);
- res1.getPlans().get(1).markers.get("a").startMarker.equals(r.markers.get("a").startMarker);
-
- FilterPlan res2 = res1.or(r);
- Assert.assertEquals(3, res2.getPlans().size());
- }
-
- /**
- * Test MultiScanPlan OR
- */
- @Test
- public void testMultiScanPlanOr() {
-
- MultiScanPlan l = createMultiScanPlan(new ScanPlan());
- MultiScanPlan r = createMultiScanPlan(new ScanPlan());
- // verify OR of two multi plans with one plan each
- Assert.assertEquals(2, l.or(r).getPlans().size());
-
- // verify OR of multi plan with a single scanplan
- Assert.assertEquals(2, l.or(new ScanPlan()).getPlans().size());
- Assert.assertEquals(2, (new ScanPlan()).or(l).getPlans().size());
-
- // verify or of two multiplans with more than one scan plan
- r = createMultiScanPlan(new ScanPlan(), new ScanPlan());
- Assert.assertEquals(3, l.or(r).getPlans().size());
- Assert.assertEquals(3, r.or(l).getPlans().size());
-
- }
-
- private MultiScanPlan createMultiScanPlan(ScanPlan... scanPlans) {
- return new MultiScanPlan(Arrays.asList(scanPlans));
- }
-
- /**
- * Test MultiScanPlan AND
- */
- @Test
- public void testMultiScanPlanAnd() {
- MultiScanPlan l = createMultiScanPlan(new ScanPlan());
- MultiScanPlan r = createMultiScanPlan(new ScanPlan());
-
- // two MultiScanPlan with single scan plans should result in new FilterPlan
- // with just one scan
- Assert.assertEquals(1, l.and(r).getPlans().size());
-
- // l has one ScanPlan, r has two. AND result should have two
- r = createMultiScanPlan(new ScanPlan(), new ScanPlan());
- Assert.assertEquals(2, l.and(r).getPlans().size());
- Assert.assertEquals(2, r.and(l).getPlans().size());
-
- // l has 2 ScanPlans, r has 3. AND result should have 6
- l = createMultiScanPlan(new ScanPlan(), new ScanPlan());
- r = createMultiScanPlan(new ScanPlan(), new ScanPlan(), new ScanPlan());
- Assert.assertEquals(6, l.and(r).getPlans().size());
- Assert.assertEquals(6, r.and(l).getPlans().size());
- }
-
- /**
- * Test plan generation from LeafNode
- *
- * @throws MetaException
- */
- @Test
- public void testLeafNodePlan() throws MetaException {
-
- final String KEY = "k1";
- final String VAL = "v1";
- final String OTHERKEY = "k2";
- LeafNode l = new LeafNode();
- l.keyName = KEY;
- l.value = VAL;
- final ScanMarker DEFAULT_SCANMARKER = null;
- List<FieldSchema> parts = new ArrayList<FieldSchema>();
- parts.add(new FieldSchema(KEY, "int", null));
- parts.add(new FieldSchema(OTHERKEY, "int", null));
-
- l.operator = Operator.EQUALS;
- verifyPlan(l, parts, KEY, new ScanMarker(VAL, INCLUSIVE, "int"), new ScanMarker(VAL, INCLUSIVE, "int"));
-
- l.operator = Operator.GREATERTHAN;
- verifyPlan(l, parts, KEY, new ScanMarker(VAL, !INCLUSIVE, "int"), DEFAULT_SCANMARKER);
-
- l.operator = Operator.GREATERTHANOREQUALTO;
- verifyPlan(l, parts, KEY, new ScanMarker(VAL, INCLUSIVE, "int"), DEFAULT_SCANMARKER);
-
- l.operator = Operator.LESSTHAN;
- verifyPlan(l, parts, KEY, DEFAULT_SCANMARKER, new ScanMarker(VAL, !INCLUSIVE, "int"));
-
- l.operator = Operator.LESSTHANOREQUALTO;
- verifyPlan(l, parts, KEY, DEFAULT_SCANMARKER, new ScanMarker(VAL, INCLUSIVE, "int"));
-
- // following leaf node plans should currently have true for 'has unsupported condition',
- // because of the condition is not on first key
- l.operator = Operator.EQUALS;
- verifyPlan(l, parts, OTHERKEY, DEFAULT_SCANMARKER, DEFAULT_SCANMARKER, false);
-
- // if tree is null, it should return equivalent of full scan, and true
- // for 'has unsupported condition'
- verifyPlan(null, parts, KEY, DEFAULT_SCANMARKER, DEFAULT_SCANMARKER, true);
-
- }
-
- private void verifyPlan(TreeNode l, List<FieldSchema> parts, String keyName, ScanMarker startMarker, ScanMarker endMarker)
- throws MetaException {
- verifyPlan(l, parts, keyName, startMarker, endMarker, false);
- }
-
- private void verifyPlan(TreeNode l, List<FieldSchema> parts, String keyName, ScanMarker startMarker, ScanMarker endMarker,
- boolean hasUnsupportedCondition) throws MetaException {
- ExpressionTree e = null;
- if (l != null) {
- e = new ExpressionTree();
- e.setRootForTest(l);
- }
- PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
- FilterPlan plan = planRes.plan;
- Assert.assertEquals("Has unsupported condition", hasUnsupportedCondition,
- planRes.hasUnsupportedCondition);
- Assert.assertEquals(1, plan.getPlans().size());
- ScanPlan splan = plan.getPlans().get(0);
- if (startMarker != null) {
- Assert.assertEquals(startMarker, splan.markers.get(keyName).startMarker);
- } else {
- Assert.assertTrue(splan.markers.get(keyName)==null ||
- splan.markers.get(keyName).startMarker==null);
- }
- if (endMarker != null) {
- Assert.assertEquals(endMarker, splan.markers.get(keyName).endMarker);
- } else {
- Assert.assertTrue(splan.markers.get(keyName)==null ||
- splan.markers.get(keyName).endMarker==null);
- }
- }
-
- /**
- * Test plan generation from TreeNode
- *
- * @throws MetaException
- */
- @Test
- public void testTreeNodePlan() throws MetaException {
-
- final String KEY = "k1";
- final String VAL1 = "10";
- final String VAL2 = "11";
- LeafNode l = new LeafNode();
- l.keyName = KEY;
- l.value = VAL1;
- final ScanMarker DEFAULT_SCANMARKER = null;
-
- List<FieldSchema> parts = new ArrayList<FieldSchema>();
- parts.add(new FieldSchema("k1", "int", null));
-
- LeafNode r = new LeafNode();
- r.keyName = KEY;
- r.value = VAL2;
-
- TreeNode tn = new TreeNode(l, LogicalOperator.AND, r);
-
- // verify plan for - k1 >= '10' and k1 < '11'
- l.operator = Operator.GREATERTHANOREQUALTO;
- r.operator = Operator.LESSTHAN;
- verifyPlan(tn, parts, KEY, new ScanMarker(VAL1, INCLUSIVE, "int"), new ScanMarker(VAL2,
- !INCLUSIVE, "int"));
-
- // verify plan for - k1 >= '10' and k1 > '11'
- l.operator = Operator.GREATERTHANOREQUALTO;
- r.operator = Operator.GREATERTHAN;
- verifyPlan(tn, parts, KEY, new ScanMarker(VAL2, !INCLUSIVE, "int"), DEFAULT_SCANMARKER);
-
- // verify plan for - k1 >= '10' or k1 > '11'
- tn = new TreeNode(l, LogicalOperator.OR, r);
- ExpressionTree e = new ExpressionTree();
- e.setRootForTest(tn);
- PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
- Assert.assertEquals(2, planRes.plan.getPlans().size());
- Assert.assertEquals(false, planRes.hasUnsupportedCondition);
-
- // verify plan for - k1 >= '10' and (k1 >= '10' or k1 > '11')
- TreeNode tn2 = new TreeNode(l, LogicalOperator.AND, tn);
- e = new ExpressionTree();
- e.setRootForTest(tn2);
- planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
- Assert.assertEquals(2, planRes.plan.getPlans().size());
- Assert.assertEquals(false, planRes.hasUnsupportedCondition);
-
- // verify plan for (k1 >= '10' and (k1 >= '10' or k1 > '11')) or k1 LIKE '2'
- // plan should return true for hasUnsupportedCondition
- LeafNode klike = new LeafNode();
- klike.keyName = KEY;
- klike.value = VAL1;
- klike.operator = Operator.LIKE;
- TreeNode tn3 = new TreeNode(tn2, LogicalOperator.OR, klike);
- e = new ExpressionTree();
- e.setRootForTest(tn3);
- planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
- Assert.assertEquals(3, planRes.plan.getPlans().size());
- Assert.assertEquals(false, planRes.hasUnsupportedCondition);
-
-
- }
-
- @Test
- public void testPartitionKeyScannerAllString() throws Exception {
- List<FieldSchema> parts = new ArrayList<FieldSchema>();
- parts.add(new FieldSchema("year", "string", null));
- parts.add(new FieldSchema("month", "string", null));
- parts.add(new FieldSchema("state", "string", null));
-
- // One prefix key and one minor key range
- ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
- PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
- ScanPlan sp = planRes.plan.getPlans().get(0);
- byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- RowFilter filter = (RowFilter)sp.getFilter(parts);
-
- // scan range contains the major key year, rowfilter contains minor key state
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
- Assert.assertFalse(Bytes.contains(startRowSuffix, "CA".getBytes()));
- Assert.assertFalse(Bytes.contains(endRowSuffix, "CA".getBytes()));
-
- PartitionKeyComparator comparator = (PartitionKeyComparator)filter.getComparator();
- Assert.assertEquals(comparator.ranges.size(), 1);
- Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
-
- // Two prefix key and one LIKE operator
- exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and month > 10 "
- + "and month <= 11 and state like 'C%'").tree;
- planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
- sp = planRes.plan.getPlans().get(0);
- startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- filter = (RowFilter)sp.getFilter(parts);
-
- // scan range contains the major key value year/month, rowfilter contains LIKE operator
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
- Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "11".getBytes()));
-
- comparator = (PartitionKeyComparator)filter.getComparator();
- Assert.assertEquals(comparator.ops.size(), 1);
- Assert.assertEquals(comparator.ops.get(0).keyName, "state");
-
- // One prefix key, one minor key range and one LIKE operator
- exprTree = PartFilterExprUtil.getFilterParser("year >= 2014 and month > 10 "
- + "and month <= 11 and state like 'C%'").tree;
- planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
- sp = planRes.plan.getPlans().get(0);
- startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- filter = (RowFilter)sp.getFilter(parts);
-
- // scan range contains the major key value year (low bound), rowfilter contains minor key state
- // and LIKE operator
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
-
- comparator = (PartitionKeyComparator)filter.getComparator();
- Assert.assertEquals(comparator.ranges.size(), 1);
- Assert.assertEquals(comparator.ranges.get(0).keyName, "month");
- Assert.assertEquals(comparator.ops.size(), 1);
- Assert.assertEquals(comparator.ops.get(0).keyName, "state");
-
- // Condition contains or
- exprTree = PartFilterExprUtil.getFilterParser("year = 2014 and (month > 10 "
- + "or month < 3)").tree;
- planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- sp = planRes.plan.getPlans().get(0);
- startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- filter = (RowFilter)sp.getFilter(parts);
-
- // The first ScanPlan contains year = 2014 and month > 10
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
- Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
-
- sp = planRes.plan.getPlans().get(1);
- startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- filter = (RowFilter)sp.getFilter(parts);
-
- // The first ScanPlan contains year = 2014 and month < 3
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "3".getBytes()));
- }
-
- @Test
- public void testPartitionKeyScannerMixedType() throws Exception {
- List<FieldSchema> parts = new ArrayList<FieldSchema>();
- parts.add(new FieldSchema("year", "int", null));
- parts.add(new FieldSchema("month", "int", null));
- parts.add(new FieldSchema("state", "string", null));
-
- // One prefix key and one minor key range
- ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
- PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
- ScanPlan sp = planRes.plan.getPlans().get(0);
- byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- RowFilter filter = (RowFilter)sp.getFilter(parts);
-
- // scan range contains the major key year, rowfilter contains minor key state
- Assert.assertTrue(Bytes.contains(startRowSuffix, Shorts.toByteArray((short)2015)));
- Assert.assertTrue(Bytes.contains(endRowSuffix, Shorts.toByteArray((short)2016)));
-
- PartitionKeyComparator comparator = (PartitionKeyComparator)filter.getComparator();
- Assert.assertEquals(comparator.ranges.size(), 1);
- Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
- }
-}
[02/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
deleted file mode 100644
index 4aa8c34..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
+++ /dev/null
@@ -1,1978 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.ResourceType;
-import org.apache.hadoop.hive.metastore.api.ResourceUri;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class TestHBaseStore {
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStore.class.getName());
- static Map<String, String> emptyParameters = new HashMap<String, String>();
- // Table with NUM_PART_KEYS partitioning keys and NUM_PARTITIONS values per key
- static final int NUM_PART_KEYS = 1;
- static final int NUM_PARTITIONS = 5;
- static final String DB = "db";
- static final String TBL = "tbl";
- static final String COL = "col";
- static final String PART_KEY_PREFIX = "part";
- static final String PART_VAL_PREFIX = "val";
- static final String PART_KV_SEPARATOR = "=";
- static final List<String> PART_KEYS = new ArrayList<String>();
- static final List<String> PART_VALS = new ArrayList<String>();
- // Initialize mock partitions
- static {
- for (int i = 1; i <= NUM_PART_KEYS; i++) {
- PART_KEYS.add(PART_KEY_PREFIX + i);
- }
- for (int i = 1; i <= NUM_PARTITIONS; i++) {
- PART_VALS.add(PART_VAL_PREFIX + i);
- }
- }
- static final long DEFAULT_TIME = System.currentTimeMillis();
- static final String PART_KEY = "part";
- static final String BOOLEAN_COL = "boolCol";
- static final String BOOLEAN_TYPE = "boolean";
- static final String BOOLEAN_VAL = "true";
- static final String LONG_COL = "longCol";
- static final String LONG_TYPE = "long";
- static final String INT_TYPE = "int";
- static final String INT_VAL = "1234";
- static final String DOUBLE_COL = "doubleCol";
- static final String DOUBLE_TYPE = "double";
- static final String DOUBLE_VAL = "3.1415";
- static final String STRING_COL = "stringCol";
- static final String STRING_TYPE = "string";
- static final String STRING_VAL = "stringval";
- static final String BINARY_COL = "binaryCol";
- static final String BINARY_TYPE = "binary";
- static final String BINARY_VAL = "1";
- static final String DECIMAL_COL = "decimalCol";
- static final String DECIMAL_TYPE = "decimal(5,3)";
- static final String DECIMAL_VAL = "12.123";
- static List<ColumnStatisticsObj> booleanColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
- static List<ColumnStatisticsObj> longColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
- static List<ColumnStatisticsObj> doubleColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
- static List<ColumnStatisticsObj> stringColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
- static List<ColumnStatisticsObj> binaryColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
- static List<ColumnStatisticsObj> decimalColStatsObjs = new ArrayList<ColumnStatisticsObj>(
- NUM_PARTITIONS);
-
- @Rule public ExpectedException thrown = ExpectedException.none();
- @Mock HTableInterface htable;
- SortedMap<String, Cell> rows = new TreeMap<>();
- HBaseStore store;
-
-
- @BeforeClass
- public static void beforeTest() {
- // All data intitializations
- populateMockStats();
- }
-
- private static void populateMockStats() {
- ColumnStatisticsObj statsObj;
- // Add NUM_PARTITIONS ColumnStatisticsObj of each type
- // For aggregate stats test, we'll treat each ColumnStatisticsObj as stats for 1 partition
- // For the rest, we'll just pick the 1st ColumnStatisticsObj from this list and use it
- for (int i = 0; i < NUM_PARTITIONS; i++) {
- statsObj = mockBooleanStats(i);
- booleanColStatsObjs.add(statsObj);
- statsObj = mockLongStats(i);
- longColStatsObjs.add(statsObj);
- statsObj = mockDoubleStats(i);
- doubleColStatsObjs.add(statsObj);
- statsObj = mockStringStats(i);
- stringColStatsObjs.add(statsObj);
- statsObj = mockBinaryStats(i);
- binaryColStatsObjs.add(statsObj);
- statsObj = mockDecimalStats(i);
- decimalColStatsObjs.add(statsObj);
- }
- }
-
- private static ColumnStatisticsObj mockBooleanStats(int i) {
- long trues = 37 + 100*i;
- long falses = 12 + 50*i;
- long nulls = 2 + i;
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(BOOLEAN_COL);
- colStatsObj.setColType(BOOLEAN_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData boolData = new BooleanColumnStatsData();
- boolData.setNumTrues(trues);
- boolData.setNumFalses(falses);
- boolData.setNumNulls(nulls);
- data.setBooleanStats(boolData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- private static ColumnStatisticsObj mockLongStats(int i) {
- long high = 120938479124L + 100*i;
- long low = -12341243213412124L - 50*i;
- long nulls = 23 + i;
- long dVs = 213L + 10*i;
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(LONG_COL);
- colStatsObj.setColType(LONG_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData longData = new LongColumnStatsData();
- longData.setHighValue(high);
- longData.setLowValue(low);
- longData.setNumNulls(nulls);
- longData.setNumDVs(dVs);
- data.setLongStats(longData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- private static ColumnStatisticsObj mockDoubleStats(int i) {
- double high = 123423.23423 + 100*i;
- double low = 0.00001234233 - 50*i;
- long nulls = 92 + i;
- long dVs = 1234123421L + 10*i;
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(DOUBLE_COL);
- colStatsObj.setColType(DOUBLE_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- DoubleColumnStatsData doubleData = new DoubleColumnStatsData();
- doubleData.setHighValue(high);
- doubleData.setLowValue(low);
- doubleData.setNumNulls(nulls);
- doubleData.setNumDVs(dVs);
- data.setDoubleStats(doubleData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- private static ColumnStatisticsObj mockStringStats(int i) {
- long maxLen = 1234 + 10*i;
- double avgLen = 32.3 + i;
- long nulls = 987 + 10*i;
- long dVs = 906 + i;
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(STRING_COL);
- colStatsObj.setColType(STRING_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- StringColumnStatsData stringData = new StringColumnStatsData();
- stringData.setMaxColLen(maxLen);
- stringData.setAvgColLen(avgLen);
- stringData.setNumNulls(nulls);
- stringData.setNumDVs(dVs);
- data.setStringStats(stringData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- private static ColumnStatisticsObj mockBinaryStats(int i) {;
- long maxLen = 123412987L + 10*i;
- double avgLen = 76.98 + i;
- long nulls = 976998797L + 10*i;
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(BINARY_COL);
- colStatsObj.setColType(BINARY_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- BinaryColumnStatsData binaryData = new BinaryColumnStatsData();
- binaryData.setMaxColLen(maxLen);
- binaryData.setAvgColLen(avgLen);
- binaryData.setNumNulls(nulls);
- data.setBinaryStats(binaryData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- private static ColumnStatisticsObj mockDecimalStats(int i) {
- Decimal high = new Decimal();
- high.setScale((short)3);
- String strHigh = String.valueOf(3876 + 100*i);
- high.setUnscaled(strHigh.getBytes());
- Decimal low = new Decimal();
- low.setScale((short)3);
- String strLow = String.valueOf(38 + i);
- low.setUnscaled(strLow.getBytes());
- long nulls = 13 + i;
- long dVs = 923947293L + 100*i;
- ColumnStatisticsObj colStatsObj = new ColumnStatisticsObj();
- colStatsObj.setColName(DECIMAL_COL);
- colStatsObj.setColType(DECIMAL_TYPE);
- ColumnStatisticsData data = new ColumnStatisticsData();
- DecimalColumnStatsData decimalData = new DecimalColumnStatsData();
- decimalData.setHighValue(high);
- decimalData.setLowValue(low);
- decimalData.setNumNulls(nulls);
- decimalData.setNumDVs(dVs);
- data.setDecimalStats(decimalData);
- colStatsObj.setStatsData(data);
- return colStatsObj;
- }
-
- @AfterClass
- public static void afterTest() {
- }
-
-
- @Before
- public void init() throws IOException {
- MockitoAnnotations.initMocks(this);
- HiveConf conf = new HiveConf();
- conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
- store = MockUtils.init(conf, htable, rows);
- }
-
- @Test
- public void createDb() throws Exception {
- String dbname = "mydb";
- Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
-
- Database d = store.getDatabase(dbname);
- Assert.assertEquals(dbname, d.getName());
- Assert.assertEquals("no description", d.getDescription());
- Assert.assertEquals("file:///tmp", d.getLocationUri());
- }
-
- @Test
- public void alterDb() throws Exception {
- String dbname = "mydb";
- Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
- db.setDescription("a description");
- store.alterDatabase(dbname, db);
-
- Database d = store.getDatabase(dbname);
- Assert.assertEquals(dbname, d.getName());
- Assert.assertEquals("a description", d.getDescription());
- Assert.assertEquals("file:///tmp", d.getLocationUri());
- }
-
- @Test
- public void dropDb() throws Exception {
- String dbname = "anotherdb";
- Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters);
- store.createDatabase(db);
-
- Database d = store.getDatabase(dbname);
- Assert.assertNotNull(d);
-
- store.dropDatabase(dbname);
- thrown.expect(NoSuchObjectException.class);
- store.getDatabase(dbname);
- }
-
- @Test
- public void createFunction() throws Exception {
- String funcName = "createfunc";
- int now = (int)(System.currentTimeMillis()/ 1000);
- Function func = new Function(funcName, DB, "o.a.h.h.myfunc", "me", PrincipalType.USER,
- now, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR,
- "file:/tmp/somewhere")));
- store.createFunction(func);
-
- Function f = store.getFunction(DB, funcName);
- Assert.assertEquals(DB, f.getDbName());
- Assert.assertEquals(funcName, f.getFunctionName());
- Assert.assertEquals("o.a.h.h.myfunc", f.getClassName());
- Assert.assertEquals("me", f.getOwnerName());
- Assert.assertEquals(PrincipalType.USER, f.getOwnerType());
- Assert.assertTrue(now <= f.getCreateTime());
- Assert.assertEquals(FunctionType.JAVA, f.getFunctionType());
- Assert.assertEquals(1, f.getResourceUrisSize());
- Assert.assertEquals(ResourceType.JAR, f.getResourceUris().get(0).getResourceType());
- Assert.assertEquals("file:/tmp/somewhere", f.getResourceUris().get(0).getUri());
- }
-
- @Test
- public void alterFunction() throws Exception {
- String funcName = "alterfunc";
- int now = (int)(System.currentTimeMillis()/ 1000);
- List<ResourceUri> uris = new ArrayList<ResourceUri>();
- uris.add(new ResourceUri(ResourceType.FILE, "whatever"));
- Function func = new Function(funcName, DB, "o.a.h.h.myfunc", "me", PrincipalType.USER,
- now, FunctionType.JAVA, uris);
- store.createFunction(func);
-
- Function f = store.getFunction(DB, funcName);
- Assert.assertEquals(ResourceType.FILE, f.getResourceUris().get(0).getResourceType());
-
- func.addToResourceUris(new ResourceUri(ResourceType.ARCHIVE, "file"));
- store.alterFunction(DB, funcName, func);
-
- f = store.getFunction(DB, funcName);
- Assert.assertEquals(2, f.getResourceUrisSize());
- Assert.assertEquals(ResourceType.FILE, f.getResourceUris().get(0).getResourceType());
- Assert.assertEquals(ResourceType.ARCHIVE, f.getResourceUris().get(1).getResourceType());
-
- }
-
- @Test
- public void dropFunction() throws Exception {
- String funcName = "delfunc";
- int now = (int)(System.currentTimeMillis()/ 1000);
- Function func = new Function(funcName, DB, "o.a.h.h.myfunc", "me", PrincipalType.USER,
- now, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR, "file:/tmp/somewhere")));
- store.createFunction(func);
-
- Function f = store.getFunction(DB, funcName);
- Assert.assertNotNull(f);
-
- store.dropFunction(DB, funcName);
- //thrown.expect(NoSuchObjectException.class);
- Assert.assertNull(store.getFunction(DB, funcName));
- }
-
- @Test
- public void createTable() throws Exception {
- String tableName = "mytable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", ""));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- Map<String, String> params = new HashMap<String, String>();
- params.put("key", "value");
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
- serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- Table t = store.getTable("default", tableName);
- Assert.assertEquals(1, t.getSd().getColsSize());
- Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
- Assert.assertEquals("", t.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp", t.getSd().getLocation());
- Assert.assertEquals("input", t.getSd().getInputFormat());
- Assert.assertEquals("output", t.getSd().getOutputFormat());
- Assert.assertFalse(t.getSd().isCompressed());
- Assert.assertEquals(17, t.getSd().getNumBuckets());
- Assert.assertEquals(1, t.getSd().getBucketColsSize());
- Assert.assertEquals("bucketcol", t.getSd().getBucketCols().get(0));
- Assert.assertEquals(1, t.getSd().getSortColsSize());
- Assert.assertEquals("sortcol", t.getSd().getSortCols().get(0).getCol());
- Assert.assertEquals(1, t.getSd().getSortCols().get(0).getOrder());
- Assert.assertEquals(1, t.getSd().getParametersSize());
- Assert.assertEquals("value", t.getSd().getParameters().get("key"));
- Assert.assertEquals("me", t.getOwner());
- Assert.assertEquals("default", t.getDbName());
- Assert.assertEquals(tableName, t.getTableName());
- Assert.assertEquals(0, t.getParametersSize());
- }
-
- @Test
- public void skewInfo() throws Exception {
- String tableName = "mytable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", ""));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", true, 0,
- serde, null, null, emptyParameters);
-
- Map<List<String>, String> map = new HashMap<List<String>, String>();
- map.put(Arrays.asList("col3"), "col4");
- SkewedInfo skew = new SkewedInfo(Arrays.asList("col1"), Arrays.asList(Arrays.asList("col2")),
- map);
- sd.setSkewedInfo(skew);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- Table t = store.getTable("default", tableName);
- Assert.assertEquals(1, t.getSd().getColsSize());
- Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
- Assert.assertEquals("", t.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp", t.getSd().getLocation());
- Assert.assertEquals("input", t.getSd().getInputFormat());
- Assert.assertEquals("output", t.getSd().getOutputFormat());
- Assert.assertTrue(t.getSd().isCompressed());
- Assert.assertEquals(0, t.getSd().getNumBuckets());
- Assert.assertEquals(0, t.getSd().getSortColsSize());
- Assert.assertEquals("me", t.getOwner());
- Assert.assertEquals("default", t.getDbName());
- Assert.assertEquals(tableName, t.getTableName());
- Assert.assertEquals(0, t.getParametersSize());
-
- skew = t.getSd().getSkewedInfo();
- Assert.assertNotNull(skew);
- Assert.assertEquals(1, skew.getSkewedColNamesSize());
- Assert.assertEquals("col1", skew.getSkewedColNames().get(0));
- Assert.assertEquals(1, skew.getSkewedColValuesSize());
- Assert.assertEquals("col2", skew.getSkewedColValues().get(0).get(0));
- Assert.assertEquals(1, skew.getSkewedColValueLocationMapsSize());
- Assert.assertEquals("col4", skew.getSkewedColValueLocationMaps().get(Arrays.asList("col3")));
-
- }
-
- @Test
- public void hashSd() throws Exception {
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", ""));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", true, 0,
- serde, null, null, emptyParameters);
-
- Map<List<String>, String> map = new HashMap<List<String>, String>();
- map.put(Arrays.asList("col3"), "col4");
- SkewedInfo skew = new SkewedInfo(Arrays.asList("col1"), Arrays.asList(Arrays.asList("col2")),
- map);
- sd.setSkewedInfo(skew);
-
- MessageDigest md = MessageDigest.getInstance("MD5");
- byte[] baseHash = HBaseUtils.hashStorageDescriptor(sd, md);
-
- StorageDescriptor changeSchema = new StorageDescriptor(sd);
- changeSchema.getCols().add(new FieldSchema("col2", "varchar(32)", "a comment"));
- byte[] schemaHash = HBaseUtils.hashStorageDescriptor(changeSchema, md);
- Assert.assertFalse(Arrays.equals(baseHash, schemaHash));
-
- StorageDescriptor changeLocation = new StorageDescriptor(sd);
- changeLocation.setLocation("file:/somewhere/else");
- byte[] locationHash = HBaseUtils.hashStorageDescriptor(changeLocation, md);
- Assert.assertArrayEquals(baseHash, locationHash);
- }
-
- @Test
- public void alterTable() throws Exception {
- String tableName = "alttable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- startTime += 10;
- table.setLastAccessTime(startTime);
- store.alterTable("default", tableName, table);
-
- Table t = store.getTable("default", tableName);
- Assert.assertEquals(1, t.getSd().getColsSize());
- Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
- Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp", t.getSd().getLocation());
- Assert.assertEquals("input", t.getSd().getInputFormat());
- Assert.assertEquals("output", t.getSd().getOutputFormat());
- Assert.assertEquals("me", t.getOwner());
- Assert.assertEquals("default", t.getDbName());
- Assert.assertEquals(tableName, t.getTableName());
- Assert.assertEquals(startTime, t.getLastAccessTime());
- }
-
- @Test
- public void dropTable() throws Exception {
- String tableName = "dtable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- Table t = store.getTable("default", tableName);
- Assert.assertNotNull(t);
-
- store.dropTable("default", tableName);
- Assert.assertNull(store.getTable("default", tableName));
- }
-
- @Test
- public void createPartition() throws Exception {
- String tableName = "myparttable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> vals = Arrays.asList("fred");
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=fred");
- Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Partition p = store.getPartition(DB, tableName, vals);
- Assert.assertEquals(1, p.getSd().getColsSize());
- Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
- Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
- Assert.assertEquals("input", p.getSd().getInputFormat());
- Assert.assertEquals("output", p.getSd().getOutputFormat());
- Assert.assertEquals(DB, p.getDbName());
- Assert.assertEquals(tableName, p.getTableName());
- Assert.assertEquals(1, p.getValuesSize());
- Assert.assertEquals("fred", p.getValues().get(0));
-
- Assert.assertTrue(store.doesPartitionExist(DB, tableName, vals));
- Assert.assertFalse(store.doesPartitionExist(DB, tableName, Arrays.asList("bob")));
- }
-
- @Test
- public void alterPartition() throws Exception {
- String tableName = "alterparttable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> vals = Arrays.asList("fred");
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=fred");
- Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- part.setLastAccessTime(startTime + 10);
- store.alterPartition(DB, tableName, vals, part);
-
- Partition p = store.getPartition(DB, tableName, vals);
- Assert.assertEquals(1, p.getSd().getColsSize());
- Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
- Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
- Assert.assertEquals("input", p.getSd().getInputFormat());
- Assert.assertEquals("output", p.getSd().getOutputFormat());
- Assert.assertEquals(DB, p.getDbName());
- Assert.assertEquals(tableName, p.getTableName());
- Assert.assertEquals(1, p.getValuesSize());
- Assert.assertEquals("fred", p.getValues().get(0));
- Assert.assertEquals(startTime + 10, p.getLastAccessTime());
-
- Assert.assertTrue(store.doesPartitionExist(DB, tableName, vals));
- Assert.assertFalse(store.doesPartitionExist(DB, tableName, Arrays.asList("bob")));
- }
-
- @Test
- public void getPartitions() throws Exception {
- String tableName = "manyParts";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
- for (String val : partVals) {
- List<String> vals = new ArrayList<String>();
- vals.add(val);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=" + val);
- Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Partition p = store.getPartition(DB, tableName, vals);
- Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
- }
-
- List<Partition> parts = store.getPartitions(DB, tableName, -1);
- Assert.assertEquals(5, parts.size());
- String[] pv = new String[5];
- for (int i = 0; i < 5; i++) pv[i] = parts.get(i).getValues().get(0);
- Arrays.sort(pv);
- Assert.assertArrayEquals(pv, partVals.toArray(new String[5]));
- }
-
- @Test
- public void listGetDropPartitionNames() throws Exception {
- String tableName = "listParts";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- partCols.add(new FieldSchema("region", "string", ""));
- Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- String[][] partVals = new String[][]{{"today", "north america"}, {"tomorrow", "europe"}};
- for (String[] pv : partVals) {
- List<String> vals = new ArrayList<String>();
- for (String v : pv) vals.add(v);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=" + pv[0] + "/region=" + pv[1]);
- Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
- }
-
- List<String> names = store.listPartitionNames(DB, tableName, (short) -1);
- Assert.assertEquals(2, names.size());
- String[] resultNames = names.toArray(new String[names.size()]);
- Arrays.sort(resultNames);
- Assert.assertArrayEquals(resultNames, new String[]{"pc=today/region=north america",
- "pc=tomorrow/region=europe"});
-
- List<Partition> parts = store.getPartitionsByNames(DB, tableName, names);
- Assert.assertArrayEquals(partVals[0], parts.get(0).getValues().toArray(new String[2]));
- Assert.assertArrayEquals(partVals[1], parts.get(1).getValues().toArray(new String[2]));
-
- store.dropPartitions(DB, tableName, names);
- List<Partition> afterDropParts = store.getPartitions(DB, tableName, -1);
- Assert.assertEquals(0, afterDropParts.size());
- }
-
-
- @Test
- public void dropPartition() throws Exception {
- String tableName = "myparttable2";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> vals = Arrays.asList("fred");
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=fred");
- Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Assert.assertNotNull(store.getPartition(DB, tableName, vals));
- store.dropPartition(DB, tableName, vals);
- thrown.expect(NoSuchObjectException.class);
- store.getPartition(DB, tableName, vals);
- }
-
- @Test
- public void createIndex() throws Exception {
- String tableName = "mytable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", ""));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- Map<String, String> params = new HashMap<String, String>();
- params.put("key", "value");
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
- serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- String indexName = "myindex";
- String indexTableName = tableName + "__" + indexName + "__";
- Index index = new Index(indexName, null, "default", tableName, startTime, startTime,
- indexTableName, sd, emptyParameters, false);
- store.addIndex(index);
-
- Index ind = store.getIndex("default", tableName, indexName);
- Assert.assertEquals(1, ind.getSd().getColsSize());
- Assert.assertEquals("col1", ind.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", ind.getSd().getCols().get(0).getType());
- Assert.assertEquals("", ind.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", ind.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", ind.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp", ind.getSd().getLocation());
- Assert.assertEquals("input", ind.getSd().getInputFormat());
- Assert.assertEquals("output", ind.getSd().getOutputFormat());
- Assert.assertFalse(ind.getSd().isCompressed());
- Assert.assertEquals(17, ind.getSd().getNumBuckets());
- Assert.assertEquals(1, ind.getSd().getBucketColsSize());
- Assert.assertEquals("bucketcol", ind.getSd().getBucketCols().get(0));
- Assert.assertEquals(1, ind.getSd().getSortColsSize());
- Assert.assertEquals("sortcol", ind.getSd().getSortCols().get(0).getCol());
- Assert.assertEquals(1, ind.getSd().getSortCols().get(0).getOrder());
- Assert.assertEquals(1, ind.getSd().getParametersSize());
- Assert.assertEquals("value", ind.getSd().getParameters().get("key"));
- Assert.assertEquals(indexName, ind.getIndexName());
- Assert.assertNull(ind.getIndexHandlerClass());
- Assert.assertEquals("default", ind.getDbName());
- Assert.assertEquals(tableName, ind.getOrigTableName());
- Assert.assertEquals(0, ind.getParametersSize());
- Assert.assertEquals(startTime, ind.getCreateTime());
- Assert.assertEquals(startTime, ind.getLastAccessTime());
- Assert.assertEquals(false, ind.isDeferredRebuild());
- }
-
- @Test
- public void alterIndex() throws Exception {
- String tableName = "mytable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", ""));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- Map<String, String> params = new HashMap<String, String>();
- params.put("key", "value");
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
- serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- String indexName = "myindex";
- Index index = new Index(indexName, null, "default", tableName, startTime, startTime,
- tableName + "__" + indexName + "__", sd, emptyParameters, false);
- store.addIndex(index);
-
- startTime += 10;
- index.setLastAccessTime(startTime);
- store.alterIndex("default", tableName, indexName, index);
-
- Index ind = store.getIndex("default", tableName, indexName);
- Assert.assertEquals(1, ind.getSd().getColsSize());
- Assert.assertEquals("col1", ind.getSd().getCols().get(0).getName());
- Assert.assertEquals("int", ind.getSd().getCols().get(0).getType());
- Assert.assertEquals("", ind.getSd().getCols().get(0).getComment());
- Assert.assertEquals("serde", ind.getSd().getSerdeInfo().getName());
- Assert.assertEquals("seriallib", ind.getSd().getSerdeInfo().getSerializationLib());
- Assert.assertEquals("file:/tmp", ind.getSd().getLocation());
- Assert.assertEquals("input", ind.getSd().getInputFormat());
- Assert.assertEquals("output", ind.getSd().getOutputFormat());
- Assert.assertFalse(ind.getSd().isCompressed());
- Assert.assertEquals(17, ind.getSd().getNumBuckets());
- Assert.assertEquals(1, ind.getSd().getBucketColsSize());
- Assert.assertEquals("bucketcol", ind.getSd().getBucketCols().get(0));
- Assert.assertEquals(1, ind.getSd().getSortColsSize());
- Assert.assertEquals("sortcol", ind.getSd().getSortCols().get(0).getCol());
- Assert.assertEquals(1, ind.getSd().getSortCols().get(0).getOrder());
- Assert.assertEquals(1, ind.getSd().getParametersSize());
- Assert.assertEquals("value", ind.getSd().getParameters().get("key"));
- Assert.assertEquals(indexName, ind.getIndexName());
- Assert.assertNull(ind.getIndexHandlerClass());
- Assert.assertEquals("default", ind.getDbName());
- Assert.assertEquals(tableName, ind.getOrigTableName());
- Assert.assertEquals(0, ind.getParametersSize());
- Assert.assertEquals(startTime, ind.getLastAccessTime());
- Assert.assertEquals(false, ind.isDeferredRebuild());
- }
-
- @Test
- public void dropIndex() throws Exception {
- String tableName = "mytable";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", ""));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- Map<String, String> params = new HashMap<String, String>();
- params.put("key", "value");
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17,
- serde, Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params);
- Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- String indexName = "myindex";
- Index index = new Index(indexName, null, "default", tableName, startTime, startTime,
- tableName + "__" + indexName + "__", sd, emptyParameters, false);
- store.addIndex(index);
-
- store.dropIndex("default", tableName, indexName);
-
- }
-
- @Test
- public void createRole() throws Exception {
- int now = (int)System.currentTimeMillis()/1000;
- String roleName = "myrole";
- store.addRole(roleName, "me");
-
- Role r = store.getRole(roleName);
- Assert.assertEquals(roleName, r.getRoleName());
- Assert.assertEquals("me", r.getOwnerName());
- Assert.assertTrue(now <= r.getCreateTime());
- }
-
- @Test
- public void dropRole() throws Exception {
- String roleName = "anotherrole";
- store.addRole(roleName, "me");
-
- Role role = store.getRole(roleName);
- Assert.assertNotNull(role);
-
- store.removeRole(roleName);
- thrown.expect(NoSuchObjectException.class);
- store.getRole(roleName);
- }
-
- // Due to the way our mock stuff works, we can only insert one column at a time, so we'll test
- // each stat type separately. We'll test them together in the integration tests.
- @Test
- public void booleanTableStatistics() throws Exception {
- // Add a boolean table stats for BOOLEAN_COL to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = booleanColStatsObjs.get(0);
- BooleanColumnStatsData boolData = obj.getStatsData().getBooleanStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(BOOLEAN_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS, dataFromDB.getSetField());
- // Compare BooleanColumnStatsData
- BooleanColumnStatsData boolDataFromDB = dataFromDB.getBooleanStats();
- Assert.assertEquals(boolData.getNumTrues(), boolDataFromDB.getNumTrues());
- Assert.assertEquals(boolData.getNumFalses(), boolDataFromDB.getNumFalses());
- Assert.assertEquals(boolData.getNumNulls(), boolDataFromDB.getNumNulls());
- }
-
- @Test
- public void longTableStatistics() throws Exception {
- createMockTable(LONG_TYPE);
- // Add a long table stats for LONG_COL to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = longColStatsObjs.get(0);
- LongColumnStatsData longData = obj.getStatsData().getLongStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(LONG_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, dataFromDB.getSetField());
- // Compare LongColumnStatsData
- LongColumnStatsData longDataFromDB = dataFromDB.getLongStats();
- Assert.assertEquals(longData.getHighValue(), longDataFromDB.getHighValue());
- Assert.assertEquals(longData.getLowValue(), longDataFromDB.getLowValue());
- Assert.assertEquals(longData.getNumNulls(), longDataFromDB.getNumNulls());
- Assert.assertEquals(longData.getNumDVs(), longDataFromDB.getNumDVs());
- }
-
- @Test
- public void doubleTableStatistics() throws Exception {
- createMockTable(DOUBLE_TYPE);
- // Add a double table stats for DOUBLE_COL to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = doubleColStatsObjs.get(0);
- DoubleColumnStatsData doubleData = obj.getStatsData().getDoubleStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(DOUBLE_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, dataFromDB.getSetField());
- // Compare DoubleColumnStatsData
- DoubleColumnStatsData doubleDataFromDB = dataFromDB.getDoubleStats();
- Assert.assertEquals(doubleData.getHighValue(), doubleDataFromDB.getHighValue(), 0.01);
- Assert.assertEquals(doubleData.getLowValue(), doubleDataFromDB.getLowValue(), 0.01);
- Assert.assertEquals(doubleData.getNumNulls(), doubleDataFromDB.getNumNulls());
- Assert.assertEquals(doubleData.getNumDVs(), doubleDataFromDB.getNumDVs());
- }
-
- @Test
- public void stringTableStatistics() throws Exception {
- createMockTable(STRING_TYPE);
- // Add a string table stats for STRING_COL to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = stringColStatsObjs.get(0);
- StringColumnStatsData stringData = obj.getStatsData().getStringStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(STRING_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, dataFromDB.getSetField());
- // Compare StringColumnStatsData
- StringColumnStatsData stringDataFromDB = dataFromDB.getStringStats();
- Assert.assertEquals(stringData.getMaxColLen(), stringDataFromDB.getMaxColLen());
- Assert.assertEquals(stringData.getAvgColLen(), stringDataFromDB.getAvgColLen(), 0.01);
- Assert.assertEquals(stringData.getNumNulls(), stringDataFromDB.getNumNulls());
- Assert.assertEquals(stringData.getNumDVs(), stringDataFromDB.getNumDVs());
- }
-
- @Test
- public void binaryTableStatistics() throws Exception {
- createMockTable(BINARY_TYPE);
- // Add a binary table stats for BINARY_COL to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = binaryColStatsObjs.get(0);
- BinaryColumnStatsData binaryData = obj.getStatsData().getBinaryStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(BINARY_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.BINARY_STATS, dataFromDB.getSetField());
- // Compare BinaryColumnStatsData
- BinaryColumnStatsData binaryDataFromDB = dataFromDB.getBinaryStats();
- Assert.assertEquals(binaryData.getMaxColLen(), binaryDataFromDB.getMaxColLen());
- Assert.assertEquals(binaryData.getAvgColLen(), binaryDataFromDB.getAvgColLen(), 0.01);
- Assert.assertEquals(binaryData.getNumNulls(), binaryDataFromDB.getNumNulls());
- }
-
- @Test
- public void decimalTableStatistics() throws Exception {
- createMockTable(DECIMAL_TYPE);
- // Add a decimal table stats for DECIMAL_COL to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for table level stats
- ColumnStatisticsDesc desc = getMockTblColStatsDesc();
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = decimalColStatsObjs.get(0);
- DecimalColumnStatsData decimalData = obj.getStatsData().getDecimalStats();
- // Add to DB
- stats.addToStatsObj(obj);
- store.updateTableColumnStatistics(stats);
- // Get from DB
- ColumnStatistics statsFromDB = store.getTableColumnStatistics(DB, TBL, Arrays.asList(DECIMAL_COL));
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.getStatsDesc().getTableName());
- Assert.assertTrue(statsFromDB.getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, dataFromDB.getSetField());
- // Compare DecimalColumnStatsData
- DecimalColumnStatsData decimalDataFromDB = dataFromDB.getDecimalStats();
- Assert.assertEquals(decimalData.getHighValue(), decimalDataFromDB.getHighValue());
- Assert.assertEquals(decimalData.getLowValue(), decimalDataFromDB.getLowValue());
- Assert.assertEquals(decimalData.getNumNulls(), decimalDataFromDB.getNumNulls());
- Assert.assertEquals(decimalData.getNumDVs(), decimalDataFromDB.getNumDVs());
- }
-
- @Test
- public void booleanPartitionStatistics() throws Exception {
- createMockTableAndPartition(BOOLEAN_TYPE, BOOLEAN_VAL);
- // Add partition stats for: BOOLEAN_COL and partition: {PART_KEY, BOOLEAN_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, BOOLEAN_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = booleanColStatsObjs.get(0);
- BooleanColumnStatsData boolData = obj.getStatsData().getBooleanStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(BOOLEAN_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.BOOLEAN_STATS, dataFromDB.getSetField());
- // Compare BooleanColumnStatsData
- BooleanColumnStatsData boolDataFromDB = dataFromDB.getBooleanStats();
- Assert.assertEquals(boolData.getNumTrues(), boolDataFromDB.getNumTrues());
- Assert.assertEquals(boolData.getNumFalses(), boolDataFromDB.getNumFalses());
- Assert.assertEquals(boolData.getNumNulls(), boolDataFromDB.getNumNulls());
- }
-
- @Test
- public void longPartitionStatistics() throws Exception {
- createMockTableAndPartition(INT_TYPE, INT_VAL);
- // Add partition stats for: LONG_COL and partition: {PART_KEY, INT_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, INT_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = longColStatsObjs.get(0);
- LongColumnStatsData longData = obj.getStatsData().getLongStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(INT_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.LONG_STATS, dataFromDB.getSetField());
- // Compare LongColumnStatsData
- LongColumnStatsData longDataFromDB = dataFromDB.getLongStats();
- Assert.assertEquals(longData.getHighValue(), longDataFromDB.getHighValue());
- Assert.assertEquals(longData.getLowValue(), longDataFromDB.getLowValue());
- Assert.assertEquals(longData.getNumNulls(), longDataFromDB.getNumNulls());
- Assert.assertEquals(longData.getNumDVs(), longDataFromDB.getNumDVs());
- }
-
- @Test
- public void doublePartitionStatistics() throws Exception {
- createMockTableAndPartition(DOUBLE_TYPE, DOUBLE_VAL);
- // Add partition stats for: DOUBLE_COL and partition: {PART_KEY, DOUBLE_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, DOUBLE_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = doubleColStatsObjs.get(0);
- DoubleColumnStatsData doubleData = obj.getStatsData().getDoubleStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(DOUBLE_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.DOUBLE_STATS, dataFromDB.getSetField());
- // Compare DoubleColumnStatsData
- DoubleColumnStatsData doubleDataFromDB = dataFromDB.getDoubleStats();
- Assert.assertEquals(doubleData.getHighValue(), doubleDataFromDB.getHighValue(), 0.01);
- Assert.assertEquals(doubleData.getLowValue(), doubleDataFromDB.getLowValue(), 0.01);
- Assert.assertEquals(doubleData.getNumNulls(), doubleDataFromDB.getNumNulls());
- Assert.assertEquals(doubleData.getNumDVs(), doubleDataFromDB.getNumDVs());
- }
-
- @Test
- public void stringPartitionStatistics() throws Exception {
- createMockTableAndPartition(STRING_TYPE, STRING_VAL);
- // Add partition stats for: STRING_COL and partition: {PART_KEY, STRING_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, STRING_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = stringColStatsObjs.get(0);
- StringColumnStatsData stringData = obj.getStatsData().getStringStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(STRING_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.STRING_STATS, dataFromDB.getSetField());
- // Compare StringColumnStatsData
- StringColumnStatsData stringDataFromDB = dataFromDB.getStringStats();
- Assert.assertEquals(stringData.getMaxColLen(), stringDataFromDB.getMaxColLen());
- Assert.assertEquals(stringData.getAvgColLen(), stringDataFromDB.getAvgColLen(), 0.01);
- Assert.assertEquals(stringData.getNumNulls(), stringDataFromDB.getNumNulls());
- Assert.assertEquals(stringData.getNumDVs(), stringDataFromDB.getNumDVs());
- }
-
- @Test
- public void binaryPartitionStatistics() throws Exception {
- createMockTableAndPartition(BINARY_TYPE, BINARY_VAL);
- // Add partition stats for: BINARY_COL and partition: {PART_KEY, BINARY_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, BINARY_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = binaryColStatsObjs.get(0);
- BinaryColumnStatsData binaryData = obj.getStatsData().getBinaryStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(BINARY_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.BINARY_STATS, dataFromDB.getSetField());
- // Compare BinaryColumnStatsData
- BinaryColumnStatsData binaryDataFromDB = dataFromDB.getBinaryStats();
- Assert.assertEquals(binaryData.getMaxColLen(), binaryDataFromDB.getMaxColLen());
- Assert.assertEquals(binaryData.getAvgColLen(), binaryDataFromDB.getAvgColLen(), 0.01);
- Assert.assertEquals(binaryData.getNumNulls(), binaryDataFromDB.getNumNulls());
- }
-
- @Test
- public void decimalPartitionStatistics() throws Exception {
- createMockTableAndPartition(DECIMAL_TYPE, DECIMAL_VAL);
- // Add partition stats for: DECIMAL_COL and partition: {PART_KEY, DECIMAL_VAL} to DB
- // Because of the way our mock implementation works we actually need to not create the table
- // before we set statistics on it.
- ColumnStatistics stats = new ColumnStatistics();
- // Get a default ColumnStatisticsDesc for partition level stats
- ColumnStatisticsDesc desc = getMockPartColStatsDesc(PART_KEY, DECIMAL_VAL);
- stats.setStatsDesc(desc);
- // Get one of the pre-created ColumnStatisticsObj
- ColumnStatisticsObj obj = decimalColStatsObjs.get(0);
- DecimalColumnStatsData decimalData = obj.getStatsData().getDecimalStats();
- // Add to DB
- stats.addToStatsObj(obj);
- List<String> parVals = new ArrayList<String>();
- parVals.add(DECIMAL_VAL);
- store.updatePartitionColumnStatistics(stats, parVals);
- // Get from DB
- List<String> partNames = new ArrayList<String>();
- partNames.add(desc.getPartName());
- List<String> colNames = new ArrayList<String>();
- colNames.add(obj.getColName());
- List<ColumnStatistics> statsFromDB = store.getPartitionColumnStatistics(DB, TBL, partNames, colNames);
- // Compare ColumnStatisticsDesc
- Assert.assertEquals(1, statsFromDB.size());
- Assert.assertEquals(desc.getLastAnalyzed(), statsFromDB.get(0).getStatsDesc().getLastAnalyzed());
- Assert.assertEquals(DB, statsFromDB.get(0).getStatsDesc().getDbName());
- Assert.assertEquals(TBL, statsFromDB.get(0).getStatsDesc().getTableName());
- Assert.assertFalse(statsFromDB.get(0).getStatsDesc().isIsTblLevel());
- // Compare ColumnStatisticsObj
- Assert.assertEquals(1, statsFromDB.get(0).getStatsObjSize());
- ColumnStatisticsObj objFromDB = statsFromDB.get(0).getStatsObj().get(0);
- ColumnStatisticsData dataFromDB = objFromDB.getStatsData();
- // Compare ColumnStatisticsData
- Assert.assertEquals(ColumnStatisticsData._Fields.DECIMAL_STATS, dataFromDB.getSetField());
- // Compare DecimalColumnStatsData
- DecimalColumnStatsData decimalDataFromDB = dataFromDB.getDecimalStats();
- Assert.assertEquals(decimalData.getHighValue(), decimalDataFromDB.getHighValue());
- Assert.assertEquals(decimalData.getLowValue(), decimalDataFromDB.getLowValue());
- Assert.assertEquals(decimalData.getNumNulls(), decimalDataFromDB.getNumNulls());
- Assert.assertEquals(decimalData.getNumDVs(), decimalDataFromDB.getNumDVs());
- }
-
- @Test
- public void createTableWithPrimaryKey() throws Exception {
- String tableName = "pktable";
- String pkName = "test_pk";
- String pkColNames[] = { "col0" };
- Table table = createMultiColumnTable(tableName, "int");
-
- List<SQLPrimaryKey> pk = Arrays.asList(
- new SQLPrimaryKey(DB, tableName, pkColNames[0], 0, pkName, true, false, true));
-
- store.createTableWithConstraints(table, pk, null, null, null);
-
- pk = store.getPrimaryKeys(DB, tableName);
-
- Assert.assertNotNull(pk);
- Assert.assertEquals(1, pk.size());
- Assert.assertEquals(DB, pk.get(0).getTable_db());
- Assert.assertEquals(tableName, pk.get(0).getTable_name());
- Assert.assertEquals(pkColNames[0], pk.get(0).getColumn_name());
- Assert.assertEquals(0, pk.get(0).getKey_seq());
- Assert.assertEquals(pkName, pk.get(0).getPk_name());
- Assert.assertTrue(pk.get(0).isEnable_cstr());
- Assert.assertFalse(pk.get(0).isValidate_cstr());
- Assert.assertTrue(pk.get(0).isRely_cstr());
-
- // Drop the primary key
- store.dropConstraint(DB, tableName, pkName);
-
- pk = store.getPrimaryKeys(DB, tableName);
- Assert.assertNull(pk);
- }
-
- @Test
- public void createTableWithForeignKey() throws Exception {
- String tableName = "fktable";
- String pkTable = "pktable";
- String pkName = "test_pk";
- String fkName = "test_fk";
- String fkColNames[] = { "col0" };
- String pkColNames[] = { "pcol0" };
- Table table = createMultiColumnTable(tableName, "int");
-
- List<SQLForeignKey> fk = Arrays.asList(
- new SQLForeignKey(DB, pkTable, pkColNames[0], DB, tableName, fkColNames[0], 0, 1, 2,
- fkName, pkName, true, false, false));
-
- store.createTableWithConstraints(table, null, fk, null, null);
-
- fk = store.getForeignKeys(DB, pkTable, DB, tableName);
-
- Assert.assertNotNull(fk);
- Assert.assertEquals(1, fk.size());
- Assert.assertEquals(DB, fk.get(0).getPktable_db());
- Assert.assertEquals(pkTable, fk.get(0).getPktable_name());
- Assert.assertEquals(pkColNames[0], fk.get(0).getPkcolumn_name());
- Assert.assertEquals(DB, fk.get(0).getFktable_db());
- Assert.assertEquals(tableName, fk.get(0).getFktable_name());
- Assert.assertEquals(fkColNames[0], fk.get(0).getFkcolumn_name());
- Assert.assertEquals(0, fk.get(0).getKey_seq());
- Assert.assertEquals(1, fk.get(0).getUpdate_rule());
- Assert.assertEquals(2, fk.get(0).getDelete_rule());
- Assert.assertEquals(fkName, fk.get(0).getFk_name());
- Assert.assertEquals(pkName, fk.get(0).getPk_name());
- Assert.assertTrue(fk.get(0).isEnable_cstr());
- Assert.assertFalse(fk.get(0).isValidate_cstr());
- Assert.assertFalse(fk.get(0).isRely_cstr());
- }
-
- // Test that we can add a primary key with multiple columns
- @Test
- public void addMultiColPrimaryKey() throws Exception {
- String tableName = "mcpktable";
- String pkName = "test_pk";
- String pkColNames[] = { "col0", "col1", "col2" };
- Table table = createMultiColumnTable(tableName, "int", "varchar(32)", "decimal(10,2)");
-
- List<SQLPrimaryKey> pk = Arrays.asList(
- new SQLPrimaryKey(DB, tableName, pkColNames[1], 0, pkName, false, true, true),
- new SQLPrimaryKey(DB, tableName, pkColNames[2], 1, pkName, false, true, true)
- );
-
- store.createTable(table);
- store.addPrimaryKeys(pk);
-
- Assert.assertNotNull(pk);
- Assert.assertEquals(2, pk.size());
- SQLPrimaryKey[] sorted = pk.toArray(new SQLPrimaryKey[2]);
- Arrays.sort(sorted, new Comparator<SQLPrimaryKey>() {
- @Override
- public int compare(SQLPrimaryKey o1, SQLPrimaryKey o2) {
- return o1.getColumn_name().compareTo(o2.getColumn_name());
- }
- });
- for (int i = 0; i < 2; i++) {
- Assert.assertEquals(DB, sorted[i].getTable_db());
- Assert.assertEquals(tableName, sorted[i].getTable_name());
- Assert.assertEquals(pkColNames[i+1], sorted[i].getColumn_name());
- Assert.assertEquals(i, sorted[i].getKey_seq());
- Assert.assertEquals(pkName, sorted[i].getPk_name());
- Assert.assertFalse(sorted[i].isEnable_cstr());
- Assert.assertTrue(sorted[i].isValidate_cstr());
- Assert.assertTrue(sorted[i].isRely_cstr());
- }
-
- }
-
- // Test that we can create a foreign key with multiple columns
- @Test
- public void addMultiColForeignKey() throws Exception {
- String tableName = "mcfktable";
- String pkTable = "pktable";
- String pkName = "test_pk";
- String fkName = "test_fk";
- String fkColNames[] = { "col0", "col1", "col2" };
- String pkColNames[] = { "pcol0", "pcol1" };
- Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
- List<SQLForeignKey> fk = Arrays.asList(
- new SQLForeignKey(DB, pkTable, pkColNames[0], DB, tableName, fkColNames[1], 0, 1, 2,
- fkName, pkName, true, false, false),
- new SQLForeignKey(DB, pkTable, pkColNames[1], DB, tableName, fkColNames[2], 1, 1, 2,
- fkName, pkName, true, false, false)
- );
-
- store.createTable(table);
- store.addForeignKeys(fk);
-
- fk = store.getForeignKeys(DB, pkTable, DB, tableName);
-
- Assert.assertNotNull(fk);
- Assert.assertEquals(2, fk.size());
- SQLForeignKey[] sorted = fk.toArray(new SQLForeignKey[2]);
- Arrays.sort(sorted, new Comparator<SQLForeignKey>() {
- @Override
- public int compare(SQLForeignKey o1, SQLForeignKey o2) {
- if (o1.getFk_name().equals(o2.getFk_name())) {
- return o1.getFkcolumn_name().compareTo(o2.getFkcolumn_name());
- } else {
- return o1.getFk_name().compareTo(o2.getFk_name());
- }
- }
- });
-
- for (int i = 0; i < 2; i++) {
- Assert.assertEquals(DB, sorted[i].getPktable_db());
- Assert.assertEquals(pkTable, sorted[i].getPktable_name());
- Assert.assertEquals(pkColNames[i], sorted[i].getPkcolumn_name());
- Assert.assertEquals(DB, sorted[i].getFktable_db());
- Assert.assertEquals(tableName, sorted[i].getFktable_name());
- Assert.assertEquals(fkColNames[i+1], sorted[i].getFkcolumn_name());
- Assert.assertEquals(i, sorted[i].getKey_seq());
- Assert.assertEquals(1, sorted[i].getUpdate_rule());
- Assert.assertEquals(2, sorted[i].getDelete_rule());
- Assert.assertEquals(fkName, sorted[i].getFk_name());
- Assert.assertEquals(pkName, sorted[i].getPk_name());
- Assert.assertTrue(sorted[i].isEnable_cstr());
- Assert.assertFalse(sorted[i].isValidate_cstr());
- Assert.assertFalse(sorted[i].isRely_cstr());
- }
-
- }
-
- // Test that we can add 2 foreign keys at once
- @Test
- public void addMultiForeignKeys() throws Exception {
- String tableName = "mcfktable";
- String pkTable = "pktable";
- String pkTable2 = "pktable2";
- String pkName = "test_pk";
- String pkName2 = "test_pk2";
- String fkName = "test_fk";
- String fkName2 = "test_fk2";
- String fkColNames[] = { "col0", "col1", "col2" };
- String pkColNames[] = { "pcol0", "pcol1" };
- String pkColNames2[] = { "p2col0" };
- Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
- List<SQLForeignKey> fk = Arrays.asList(
- new SQLForeignKey(DB, pkTable, pkColNames[0], DB, tableName, fkColNames[1], 0, 1, 2,
- fkName, pkName, true, false, true),
- new SQLForeignKey(DB, pkTable, pkColNames[1], DB, tableName, fkColNames[2], 1, 1, 2,
- fkName, pkName, true, false, true),
- new SQLForeignKey(DB, pkTable2, pkColNames2[0], DB, tableName, fkColNames[0], 0, 1, 2,
- fkName2, pkName2, true, false, true)
- );
-
- store.createTable(table);
- store.addForeignKeys(fk);
-
- fk = store.getForeignKeys(DB, pkTable, DB, tableName);
-
- Assert.assertNotNull(fk);
- Assert.assertEquals(2, fk.size());
- SQLForeignKey[] sorted = fk.toArray(new SQLForeignKey[2]);
- Arrays.sort(sorted, new Comparator<SQLForeignKey>() {
- @Override
- public int compare(SQLForeignKey o1, SQLForeignKey o2) {
- if (o1.getFk_name().equals(o2.getFk_name())) {
- return o1.getFkcolumn_name().compareTo(o2.getFkcolumn_name());
- } else {
- return o1.getFk_name().compareTo(o2.getFk_name());
- }
- }
- });
-
- for (int i = 0; i < 2; i++) {
- Assert.assertEquals(DB, sorted[i].getPktable_db());
- Assert.assertEquals(pkTable, sorted[i].getPktable_name());
- Assert.assertEquals(pkColNames[i], sorted[i].getPkcolumn_name());
- Assert.assertEquals(DB, sorted[i].getFktable_db());
- Assert.assertEquals(tableName, sorted[i].getFktable_name());
- Assert.assertEquals(fkColNames[i+1], sorted[i].getFkcolumn_name());
- Assert.assertEquals(i, sorted[i].getKey_seq());
- Assert.assertEquals(1, sorted[i].getUpdate_rule());
- Assert.assertEquals(2, sorted[i].getDelete_rule());
- Assert.assertEquals(fkName, sorted[i].getFk_name());
- Assert.assertEquals(pkName, sorted[i].getPk_name());
- Assert.assertTrue(sorted[i].isEnable_cstr());
- Assert.assertFalse(sorted[i].isValidate_cstr());
- Assert.assertTrue(sorted[i].isRely_cstr());
- }
- fk = store.getForeignKeys(DB, pkTable2, DB, tableName);
- Assert.assertNotNull(fk);
- Assert.assertEquals(1, fk.size());
- Assert.assertEquals(DB, fk.get(0).getPktable_db());
- Assert.assertEquals(pkTable2, fk.get(0).getPktable_name());
- Assert.assertEquals(pkColNames2[0], fk.get(0).getPkcolumn_name());
- Assert.assertEquals(DB, fk.get(0).getFktable_db());
- Assert.assertEquals(tableName, fk.get(0).getFktable_name());
- Assert.assertEquals(fkColNames[0], fk.get(0).getFkcolumn_name());
- Assert.assertEquals(0, fk.get(0).getKey_seq());
- Assert.assertEquals(1, fk.get(0).getUpdate_rule());
- Assert.assertEquals(2, fk.get(0).getDelete_rule());
- Assert.assertEquals(fkName2, fk.get(0).getFk_name());
- Assert.assertEquals(pkName2, fk.get(0).getPk_name());
- Assert.assertTrue(fk.get(0).isEnable_cstr());
- Assert.assertFalse(fk.get(0).isValidate_cstr());
- Assert.assertTrue(fk.get(0).isRely_cstr());
-
- }
-
- // Test that we can add a foreign key when one already exists
- @Test
- public void addSecondForeignKeys() throws Exception {
- String tableName = "mcfktable";
- String pkTable = "pktable";
- String pkTable2 = "pktable2";
- String pkName = "test_pk";
- String pkName2 = "test_pk2";
- String fkName = "test_fk";
- String fkName2 = "test_fk2";
- String fkColNames[] = { "col0", "col1", "col2" };
- String pkColNames[] = { "pcol0", "pcol1" };
- String pkColNames2[] = { "p2col0" };
- Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
- List<SQLForeignKey> fk = Arrays.asList(
- new SQLForeignKey(DB, pkTable, pkColNames[0], DB, tableName, fkColNames[1], 0, 1, 2,
- fkName, pkName, true, false, true),
- new SQLForeignKey(DB, pkTable, pkColNames[1], DB, tableName, fkColNames[2], 1, 1, 2,
- fkName, pkName, true, false, true)
- );
-
- store.createTable(table);
- store.addForeignKeys(fk);
-
- fk = Arrays.asList(
- new SQLForeignKey(DB, pkTable2, pkColNames2[0], DB, tableName, fkColNames[0], 0, 1, 2,
- fkName2, pkName2, true, false, true)
- );
- store.addForeignKeys(fk);
-
- fk = store.getForeignKeys(DB, pkTable, DB, tableName);
-
- Assert.assertNotNull(fk);
- Assert.assertEquals(2, fk.size());
- SQLForeignKey[] sorted = fk.toArray(new SQLForeignKey[2]);
- Arrays.sort(sorted, new Comparator<SQLForeignKey>() {
- @Override
- public int compare(SQLForeignKey o1, SQLForeignKey o2) {
- if (o1.getFk_name().equals(o2.getFk_name())) {
- return o1.getFkcolumn_name().compareTo(o2.getFkcolumn_name());
- } else {
- return o1.getFk_name().compareTo(o2.getFk_name());
- }
- }
- });
-
- for (int i = 0; i < 2; i++) {
- Assert.assertEquals(DB, sorted[i].getPktable_db());
- Assert.assertEquals(pkTable, sorted[i].getPktable_name());
- Assert.assertEquals(pkColNames[i], sorted[i].getPkcolumn_name());
- Assert.assertEquals(DB, sorted[i].getFktable_db());
- Assert.assertEquals(tableName, sorted[i].getFktable_name());
- Assert.assertEquals(fkColNames[i+1], sorted[i].getFkcolumn_name());
- Assert.assertEquals(i, sorted[i].getKey_seq());
- Assert.assertEquals(1, sorted[i].getUpdate_rule());
- Assert.assertEquals(2, sorted[i].getDelete_rule());
- Assert.assertEquals(fkName, sorted[i].getFk_name());
- Assert.assertEquals(pkName, sorted[i].getPk_name());
- Assert.assertTrue(sorted[i].isEnable_cstr());
- Assert.assertFalse(sorted[i].isValidate_cstr());
- Assert.assertTrue(sorted[i].isRely_cstr());
- }
-
- fk = store.getForeignKeys(DB, pkTable2, DB, tableName);
- Assert.assertNotNull(fk);
- Assert.assertEquals(1, fk.size());
- Assert.assertEquals(DB, fk.get(0).getPktable_db());
- Assert.assertEquals(pkTable2, fk.get(0).getPktable_name());
- Assert.assertEquals(pkColNames2[0], fk.get(0).getPkcolumn_name());
- Assert.assertEquals(DB, fk.get(0).getFktable_db());
- Assert.assertEquals(tableName, fk.get(0).getFktable_name());
- Assert.assertEquals(fkColNames[0], fk.get(0).getFkcolumn_name());
- Assert.assertEquals(0, fk.get(0).getKey_seq());
- Assert.assertEquals(1, fk.get(0).getUpdate_rule());
- Assert.assertEquals(2, fk.get(0).getDelete_rule());
- Assert.assertEquals(fkName2, fk.get(0).getFk_name());
- Assert.assertEquals(pkName2, fk.get(0).getPk_name());
- Assert.assertTrue(fk.get(0).isEnable_cstr());
- Assert.assertFalse(fk.get(0).isValidate_cstr());
- Assert.assertTrue(fk.get(0).isRely_cstr());
-
- // Check that passing null gets all the foreign keys
- fk = store.getForeignKeys(null, null, DB, tableName);
- Assert.assertNotNull(fk);
- Assert.assertEquals(3, fk.size());
-
- store.dropConstraint(DB, tableName, fkName);
-
- fk = store.getForeignKeys(DB, pkTable2, DB, tableName);
- Assert.assertNotNull(fk);
- Assert.assertEquals(1, fk.size());
- Assert.assertEquals(DB, fk.get(0).getPktable_db());
- Assert.assertEquals(pkTable2, fk.get(0).getPktable_name());
- Assert.assertEquals(pkColNames2[0], fk.get(0).getPkcolumn_name());
- Assert.assertEquals(DB, fk.get(0).getFktable_db());
- Assert.assertEquals(tableName, fk.get(0).getFktable_name());
- Assert.assertEquals(fkColNames[0], fk.get(0).getFkcolumn_name());
- Assert.assertEquals(0, fk.get(0).getKey_seq());
- Assert.assertEquals(1, fk.get(0).getUpdate_rule());
- Assert.assertEquals(2, fk.get(0).getDelete_rule());
- Assert.assertEquals(fkName2, fk.get(0).getFk_name());
- Assert.assertEquals(pkName2, fk.get(0).getPk_name());
- Assert.assertTrue(fk.get(0).isEnable_cstr());
- Assert.assertFalse(fk.get(0).isValidate_cstr());
- Assert.assertTrue(fk.get(0).isRely_cstr());
-
- store.dropConstraint(DB, tableName, fkName2);
-
- fk = store.getForeignKeys(DB, pkTable2, DB, tableName);
- Assert.assertNull(fk);
- }
-
- // Try adding a primary key when one already exists
- @Test(expected= MetaException.class)
- public void doublePrimaryKey() throws Exception {
- String tableName = "pktable";
- String pkName = "test_pk";
- String pkColNames[] = { "col0" };
- Table table = createMultiColumnTable(tableName, "int");
-
- List<SQLPrimaryKey> pk = Arrays.asList(
- new SQLPrimaryKey(DB, tableName, pkColNames[0], 0, pkName, true, false, true));
-
- store.createTableWithConstraints(table, pk, null, null, null);
-
- store.addPrimaryKeys(pk);
- }
-
- @Test
- public void createTableWithUniqueConstraint() throws Exception {
- String tableName = "uktable";
- String ukName = "test_uk";
- String ukColNames[] = { "col0" };
- Table table = createMultiColumnTable(tableName, "int");
-
- List<SQLUniqueConstraint> uk = Arrays.asList(
- new SQLUniqueConstraint(DB, tableName, ukColNames[0], 0, ukName, true, false, true));
-
- store.createTableWithConstraints(table, null, null, uk, null);
-
- uk = store.getUniqueConstraints(DB, tableName);
-
- Assert.assertNotNull(uk);
- Assert.assertEquals(1, uk.size());
- Assert.assertEquals(DB, uk.get(0).getTable_db());
- Assert.assertEquals(tableName, uk.get(0).getTable_name());
- Assert.assertEquals(ukColNames[0], uk.get(0).getColumn_name());
- Assert.assertEquals(0, uk.get(0).getKey_seq());
- Assert.assertEquals(ukName, uk.get(0).getUk_name());
- Assert.assertTrue(uk.get(0).isEnable_cstr());
- Assert.assertFalse(uk.get(0).isValidate_cstr());
- Assert.assertTrue(uk.get(0).isRely_cstr());
-
- // Drop the unique constraint
- store.dropConstraint(DB, tableName, ukName);
-
- uk = store.getUniqueConstraints(DB, tableName);
- Assert.assertNull(uk);
- }
-
- @Test
- public void addMultiUniqueConstraints() throws Exception {
- String tableName = "mcuktable";
- String ukName = "test_uk";
- String ukName2 = "test_uk2";
- String ukColNames[] = { "col0", "col1" };
- Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
- List<SQLUniqueConstraint> uks = Arrays.asList(
- new SQLUniqueConstraint(DB, tableName, ukColNames[0], 0, ukName, true, false, true),
- new SQLUniqueConstraint(DB, tableName, ukColNames[1], 0, ukName2, true, false, true)
- );
-
- store.createTable(table);
- store.addUniqueConstraints(uks);
-
- uks = store.getUniqueConstraints(DB, tableName);
-
- Assert.assertNotNull(uks);
- Assert.assertEquals(2, uks.size());
- SQLUniqueConstraint[] sorted = uks.toArray(new SQLUniqueConstraint[2]);
- Arrays.sort(sorted, new Comparator<SQLUniqueConstraint>() {
- @Override
- public int compare(SQLUniqueConstraint o1, SQLUniqueConstraint o2) {
- if (o1.getUk_name().equals(o2.getUk_name())) {
- return o1.getColumn_name().compareTo(o2.getColumn_name());
- } else {
- return o1.getUk_name().compareTo(o2.getUk_name());
- }
- }
- });
-
- Assert.assertEquals(DB, sorted[0].getTable_db());
- Assert.assertEquals(tableName, sorted[0].getTable_name());
- Assert.assertEquals(ukColNames[0], sorted[0].getColumn_name());
- Assert.assertEquals(0, sorted[0].getKey_seq());
- Assert.assertEquals(ukName, sorted[0].getUk_name());
- Assert.assertTrue(sorted[0].isEnable_cstr());
- Assert.assertFalse(sorted[0].isValidate_cstr());
- Assert.assertTrue(sorted[0].isRely_cstr());
-
- Assert.assertEquals(DB, sorted[1].getTable_db());
- Assert.assertEquals(tableName, sorted[1].getTable_name());
- Assert.assertEquals(ukColNames[1], sorted[1].getColumn_name());
- Assert.assertEquals(0, sorted[1].getKey_seq());
- Assert.assertEquals(ukName2, sorted[1].getUk_name());
- Assert.assertTrue(sorted[1].isEnable_cstr());
- Assert.assertFalse(sorted[1].isValidate_cstr());
- Assert.assertTrue(sorted[1].isRely_cstr());
- }
-
- @Test
- public void addMultiNotNullConstraints() throws Exception {
- String tableName = "mcnntable";
- String nnName = "test_nn";
- String nnName2 = "test_nn2";
- String nnColNames[] = { "col0", "col1" };
- Table table = createMultiColumnTable(tableName, "int", "double", "timestamp");
-
- List<SQLNotNullConstraint> nns = Arrays.asList(
- new SQLNotNullConstraint(DB, tableName, nnColNames[0], nnName, true, false, t
<TRUNCATED>
[10/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
deleted file mode 100644
index ab6457e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ /dev/null
@@ -1,2942 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterators;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.RegexStringComparator;
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hive.common.ObjectPair;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.hbase.PartitionKeyComparator.Operator;
-import org.apache.hive.common.util.BloomFilter;
-import org.apache.thrift.TBase;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.protocol.TSimpleJSONProtocol;
-import org.apache.thrift.transport.TMemoryBuffer;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-
-
-/**
- * Class to manage storing object in and reading them from HBase.
- */
-public class HBaseReadWrite implements MetadataStore {
-
- final static String AGGR_STATS_TABLE = "HBMS_AGGR_STATS";
- final static String DB_TABLE = "HBMS_DBS";
- final static String FUNC_TABLE = "HBMS_FUNCS";
- final static String GLOBAL_PRIVS_TABLE = "HBMS_GLOBAL_PRIVS";
- final static String PART_TABLE = "HBMS_PARTITIONS";
- final static String ROLE_TABLE = "HBMS_ROLES";
- final static String SD_TABLE = "HBMS_SDS";
- final static String SECURITY_TABLE = "HBMS_SECURITY";
- final static String SEQUENCES_TABLE = "HBMS_SEQUENCES";
- final static String TABLE_TABLE = "HBMS_TBLS";
- final static String INDEX_TABLE = "HBMS_INDEX";
- final static String USER_TO_ROLE_TABLE = "HBMS_USER_TO_ROLE";
- final static String FILE_METADATA_TABLE = "HBMS_FILE_METADATA";
- final static byte[] CATALOG_CF = "c".getBytes(HBaseUtils.ENCODING);
- final static byte[] STATS_CF = "s".getBytes(HBaseUtils.ENCODING);
- final static String NO_CACHE_CONF = "no.use.cache";
- /**
- * List of tables in HBase
- */
- public final static String[] tableNames = { AGGR_STATS_TABLE, DB_TABLE, FUNC_TABLE,
- GLOBAL_PRIVS_TABLE, PART_TABLE, USER_TO_ROLE_TABLE,
- ROLE_TABLE, SD_TABLE, SECURITY_TABLE, SEQUENCES_TABLE,
- TABLE_TABLE, INDEX_TABLE, FILE_METADATA_TABLE };
- public final static Map<String, List<byte[]>> columnFamilies = new HashMap<> (tableNames.length);
-
- static {
- columnFamilies.put(AGGR_STATS_TABLE, Arrays.asList(CATALOG_CF));
- columnFamilies.put(DB_TABLE, Arrays.asList(CATALOG_CF));
- columnFamilies.put(FUNC_TABLE, Arrays.asList(CATALOG_CF));
- columnFamilies.put(GLOBAL_PRIVS_TABLE, Arrays.asList(CATALOG_CF));
- columnFamilies.put(PART_TABLE, Arrays.asList(CATALOG_CF, STATS_CF));
- columnFamilies.put(USER_TO_ROLE_TABLE, Arrays.asList(CATALOG_CF));
- columnFamilies.put(ROLE_TABLE, Arrays.asList(CATALOG_CF));
- columnFamilies.put(SD_TABLE, Arrays.asList(CATALOG_CF));
- columnFamilies.put(SECURITY_TABLE, Arrays.asList(CATALOG_CF));
- columnFamilies.put(SEQUENCES_TABLE, Arrays.asList(CATALOG_CF));
- columnFamilies.put(TABLE_TABLE, Arrays.asList(CATALOG_CF, STATS_CF));
- columnFamilies.put(INDEX_TABLE, Arrays.asList(CATALOG_CF, STATS_CF));
- // Stats CF will contain PPD stats.
- columnFamilies.put(FILE_METADATA_TABLE, Arrays.asList(CATALOG_CF, STATS_CF));
- }
-
- final static byte[] MASTER_KEY_SEQUENCE = "master_key".getBytes(HBaseUtils.ENCODING);
- // The change version functionality uses the sequences table, but we don't want to give the
- // caller complete control over the sequence name as they might inadvertently clash with one of
- // our sequence keys, so add a prefix to their topic name.
-
- final static byte[] AGGR_STATS_BLOOM_COL = "b".getBytes(HBaseUtils.ENCODING);
- private final static byte[] AGGR_STATS_STATS_COL = "s".getBytes(HBaseUtils.ENCODING);
- private final static byte[] CATALOG_COL = "c".getBytes(HBaseUtils.ENCODING);
- private final static byte[] ROLES_COL = "roles".getBytes(HBaseUtils.ENCODING);
- private final static byte[] REF_COUNT_COL = "ref".getBytes(HBaseUtils.ENCODING);
- private final static byte[] DELEGATION_TOKEN_COL = "dt".getBytes(HBaseUtils.ENCODING);
- private final static byte[] MASTER_KEY_COL = "mk".getBytes(HBaseUtils.ENCODING);
- private final static byte[] PRIMARY_KEY_COL = "pk".getBytes(HBaseUtils.ENCODING);
- private final static byte[] FOREIGN_KEY_COL = "fk".getBytes(HBaseUtils.ENCODING);
- private final static byte[] UNIQUE_CONSTRAINT_COL = "uk".getBytes(HBaseUtils.ENCODING);
- private final static byte[] NOT_NULL_CONSTRAINT_COL = "nn".getBytes(HBaseUtils.ENCODING);
- private final static byte[] GLOBAL_PRIVS_KEY = "gp".getBytes(HBaseUtils.ENCODING);
- private final static byte[] SEQUENCES_KEY = "seq".getBytes(HBaseUtils.ENCODING);
- private final static int TABLES_TO_CACHE = 10;
- // False positives are very bad here because they cause us to invalidate entries we shouldn't.
- // Space used and # of hash functions grows in proportion to ln of num bits so a 10x increase
- // in accuracy doubles the required space and number of hash functions.
- private final static double STATS_BF_ERROR_RATE = 0.001;
-
- @VisibleForTesting final static String TEST_CONN = "test_connection";
- private static HBaseConnection testConn;
-
- static final private Logger LOG = LoggerFactory.getLogger(HBaseReadWrite.class.getName());
-
- private static ThreadLocal<HBaseReadWrite> self = new ThreadLocal<HBaseReadWrite>() {
- @Override
- protected HBaseReadWrite initialValue() {
- if (staticConf == null) {
- throw new RuntimeException("Attempt to create HBaseReadWrite with no configuration set");
- }
- return new HBaseReadWrite(staticConf);
- }
- };
-
- private static boolean tablesCreated = false;
- private static Configuration staticConf = null;
-
- private final Configuration conf;
- private HBaseConnection conn;
- private MessageDigest md;
- private ObjectCache<ObjectPair<String, String>, Table> tableCache;
- private ObjectCache<ByteArrayWrapper, StorageDescriptor> sdCache;
- private PartitionCache partCache;
- private StatsCache statsCache;
- private Counter tableHits;
- private Counter tableMisses;
- private Counter tableOverflows;
- private Counter partHits;
- private Counter partMisses;
- private Counter partOverflows;
- private Counter sdHits;
- private Counter sdMisses;
- private Counter sdOverflows;
- private List<Counter> counters;
- // roleCache doesn't use ObjectCache because I don't want to limit the size. I am assuming
- // that the number of roles will always be small (< 100) so caching the whole thing should not
- // be painful.
- private final Map<String, HbaseMetastoreProto.RoleGrantInfoList> roleCache;
- boolean entireRoleTableInCache;
-
- /**
- * Set the configuration for all HBaseReadWrite instances.
- * @param configuration Configuration object
- */
- public static synchronized void setConf(Configuration configuration) {
- if (staticConf == null) {
- staticConf = configuration;
- } else {
- LOG.info("Attempt to set conf when it has already been set.");
- }
- }
-
- /**
- * Get the instance of HBaseReadWrite for the current thread. This can only be called after
- * {@link #setConf} has been called. Woe betide you if that's not the case.
- * @return thread's instance of HBaseReadWrite
- */
- static HBaseReadWrite getInstance() {
- if (staticConf == null) {
- throw new RuntimeException("Must set conf object before getting an instance");
- }
- return self.get();
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- private HBaseReadWrite(Configuration configuration) {
- conf = configuration;
- HBaseConfiguration.addHbaseResources(conf);
-
- try {
- String connClass = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS);
- if (TEST_CONN.equals(connClass)) {
- conn = testConn;
- LOG.debug("Using test connection.");
- } else {
- LOG.debug("Instantiating connection class " + connClass);
- Class c = Class.forName(connClass);
- Object o = c.newInstance();
- if (HBaseConnection.class.isAssignableFrom(o.getClass())) {
- conn = (HBaseConnection) o;
- } else {
- throw new IOException(connClass + " is not an instance of HBaseConnection.");
- }
- conn.setConf(conf);
- conn.connect();
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- try {
- md = MessageDigest.getInstance("MD5");
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException(e);
- }
- int totalCatalogObjectsToCache =
- HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTORE_HBASE_CATALOG_CACHE_SIZE);
-
- tableHits = new Counter("table cache hits");
- tableMisses = new Counter("table cache misses");
- tableOverflows = new Counter("table cache overflows");
- partHits = new Counter("partition cache hits");
- partMisses = new Counter("partition cache misses");
- partOverflows = new Counter("partition cache overflows");
- sdHits = new Counter("storage descriptor cache hits");
- sdMisses = new Counter("storage descriptor cache misses");
- sdOverflows = new Counter("storage descriptor cache overflows");
- counters = new ArrayList<>();
- counters.add(tableHits);
- counters.add(tableMisses);
- counters.add(tableOverflows);
- counters.add(partHits);
- counters.add(partMisses);
- counters.add(partOverflows);
- counters.add(sdHits);
- counters.add(sdMisses);
- counters.add(sdOverflows);
-
- // Give 1% of catalog cache space to storage descriptors
- // (storage descriptors are shared, so 99% should be the same for a given table)
- int sdsCacheSize = totalCatalogObjectsToCache / 100;
- if (conf.getBoolean(NO_CACHE_CONF, false)) {
- tableCache = new BogusObjectCache<>();
- sdCache = new BogusObjectCache<>();
- partCache = new BogusPartitionCache();
- } else {
- tableCache = new ObjectCache<>(TABLES_TO_CACHE, tableHits, tableMisses, tableOverflows);
- sdCache = new ObjectCache<>(sdsCacheSize, sdHits, sdMisses, sdOverflows);
- partCache = new PartitionCache(totalCatalogObjectsToCache, partHits, partMisses, partOverflows);
- }
- statsCache = StatsCache.getInstance(conf);
- roleCache = new HashMap<>();
- entireRoleTableInCache = false;
- }
-
- // Synchronize this so not everyone's doing it at once.
- static synchronized void createTablesIfNotExist() throws IOException {
- if (!tablesCreated) {
- for (String name : tableNames) {
- if (self.get().conn.getHBaseTable(name, true) == null) {
- List<byte[]> families = columnFamilies.get(name);
- self.get().conn.createHBaseTable(name, families);
- }
- }
- tablesCreated = true;
- }
- }
-
- /**********************************************************************************************
- * Transaction related methods
- *********************************************************************************************/
-
- /**
- * Begin a transaction
- */
- void begin() {
- try {
- conn.beginTransaction();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Commit a transaction
- */
- void commit() {
- try {
- conn.commitTransaction();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- void rollback() {
- try {
- conn.rollbackTransaction();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- void close() throws IOException {
- conn.close();
- }
-
- /**********************************************************************************************
- * Database related methods
- *********************************************************************************************/
-
- /**
- * Fetch a database object
- * @param name name of the database to fetch
- * @return the database object, or null if there is no such database
- * @throws IOException
- */
- Database getDb(String name) throws IOException {
- byte[] key = HBaseUtils.buildKey(name);
- byte[] serialized = read(DB_TABLE, key, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return null;
- return HBaseUtils.deserializeDatabase(name, serialized);
- }
-
- /**
- * Get a list of databases.
- * @param regex Regular expression to use in searching for database names. It is expected to
- * be a Java regular expression. If it is null then all databases will be returned.
- * @return list of databases matching the regular expression.
- * @throws IOException
- */
- List<Database> scanDatabases(String regex) throws IOException {
- Filter filter = null;
- if (regex != null) {
- filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
- }
- Iterator<Result> iter =
- scan(DB_TABLE, CATALOG_CF, CATALOG_COL, filter);
- List<Database> databases = new ArrayList<>();
- while (iter.hasNext()) {
- Result result = iter.next();
- databases.add(HBaseUtils.deserializeDatabase(result.getRow(),
- result.getValue(CATALOG_CF, CATALOG_COL)));
- }
- return databases;
- }
-
- /**
- * Store a database object
- * @param database database object to store
- * @throws IOException
- */
- void putDb(Database database) throws IOException {
- byte[][] serialized = HBaseUtils.serializeDatabase(database);
- store(DB_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- }
-
- /**
- * Drop a database
- * @param name name of db to drop
- * @throws IOException
- */
- void deleteDb(String name) throws IOException {
- byte[] key = HBaseUtils.buildKey(name);
- delete(DB_TABLE, key, null, null);
- }
-
- /**
- * Print out the database. Intended for use by {@link org.apache.hadoop.hive.metastore.hbase.HBaseSchemaTool}
- * @param name name of database to print
- * @return string printout of database
- */
- String printDatabase(String name) throws IOException, TException {
- Database db = getDb(name);
- if (db == null) return noSuch(name, "database");
- else return dumpThriftObject(db);
- }
-
- /**
- * Print out databases.
- * @param regex regular to use to search for databases
- * @return databases as a string, one each
- * @throws IOException
- * @throws TException
- */
- List<String> printDatabases(String regex) throws IOException, TException {
- List<Database> dbs = scanDatabases(regex);
- if (dbs.size() == 0) {
- return noMatch(regex, "database");
- } else {
- List<String> lines = new ArrayList<>();
- for (Database db : dbs) lines.add(dumpThriftObject(db));
- return lines;
- }
- }
-
- int getDatabaseCount() throws IOException {
- Filter fil = new FirstKeyOnlyFilter();
- Iterator<Result> iter = scan(DB_TABLE, fil);
- return Iterators.size(iter);
- }
-
- /**********************************************************************************************
- * Function related methods
- *********************************************************************************************/
-
- /**
- * Fetch a function object
- * @param dbName name of the database the function is in
- * @param functionName name of the function to fetch
- * @return the function object, or null if there is no such function
- * @throws IOException
- */
- Function getFunction(String dbName, String functionName) throws IOException {
- byte[] key = HBaseUtils.buildKey(dbName, functionName);
- byte[] serialized = read(FUNC_TABLE, key, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return null;
- return HBaseUtils.deserializeFunction(dbName, functionName, serialized);
- }
-
- /**
- * Get a list of functions.
- * @param dbName Name of the database to search in.
- * @param regex Regular expression to use in searching for function names. It is expected to
- * be a Java regular expression. If it is null then all functions will be returned.
- * @return list of functions matching the regular expression.
- * @throws IOException
- */
- List<Function> scanFunctions(String dbName, String regex) throws IOException {
- byte[] keyPrefix = null;
- if (dbName != null) {
- keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName);
- }
- Filter filter = null;
- if (regex != null) {
- filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
- }
- Iterator<Result> iter =
- scan(FUNC_TABLE, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix), CATALOG_CF, CATALOG_COL, filter);
- List<Function> functions = new ArrayList<>();
- while (iter.hasNext()) {
- Result result = iter.next();
- functions.add(HBaseUtils.deserializeFunction(result.getRow(),
- result.getValue(CATALOG_CF, CATALOG_COL)));
- }
- return functions;
- }
-
- /**
- * Store a function object
- * @param function function object to store
- * @throws IOException
- */
- void putFunction(Function function) throws IOException {
- byte[][] serialized = HBaseUtils.serializeFunction(function);
- store(FUNC_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- }
-
- /**
- * Drop a function
- * @param dbName name of database the function is in
- * @param functionName name of function to drop
- * @throws IOException
- */
- void deleteFunction(String dbName, String functionName) throws IOException {
- byte[] key = HBaseUtils.buildKey(dbName, functionName);
- delete(FUNC_TABLE, key, null, null);
- }
-
- /**
- * Print out a function
- * @param key key to get the function, must include dbname.
- * @return string of the function
- * @throws IOException
- * @throws TException
- */
- String printFunction(String key) throws IOException, TException {
- byte[] k = HBaseUtils.buildKey(key);
- byte[] serialized = read(FUNC_TABLE, k, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return noSuch(key, "function");
- Function func = HBaseUtils.deserializeFunction(k, serialized);
- return dumpThriftObject(func);
- }
-
- /**
- * Print out functions
- * @param regex regular expression to use in matching functions
- * @return list of strings, one function each
- * @throws IOException
- * @throws TException
- */
- List<String> printFunctions(String regex) throws IOException, TException {
- Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
- Iterator<Result> iter = scan(FUNC_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
- List<String> lines = new ArrayList<>();
- while (iter.hasNext()) {
- Result result = iter.next();
- lines.add(dumpThriftObject(HBaseUtils.deserializeFunction(result.getRow(),
- result.getValue(CATALOG_CF, CATALOG_COL))));
- }
- if (lines.size() == 0) lines = noMatch(regex, "function");
- return lines;
- }
-
- /**********************************************************************************************
- * Global privilege related methods
- *********************************************************************************************/
-
- /**
- * Fetch the global privileges object
- * @return
- * @throws IOException
- */
- PrincipalPrivilegeSet getGlobalPrivs() throws IOException {
- byte[] key = GLOBAL_PRIVS_KEY;
- byte[] serialized = read(GLOBAL_PRIVS_TABLE, key, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return null;
- return HBaseUtils.deserializePrincipalPrivilegeSet(serialized);
- }
-
- /**
- * Store the global privileges object
- * @throws IOException
- */
- void putGlobalPrivs(PrincipalPrivilegeSet privs) throws IOException {
- byte[] key = GLOBAL_PRIVS_KEY;
- byte[] serialized = HBaseUtils.serializePrincipalPrivilegeSet(privs);
- store(GLOBAL_PRIVS_TABLE, key, CATALOG_CF, CATALOG_COL, serialized);
- }
-
- /**
- * Print out the global privileges.
- * @return string containing the global privileges
- * @throws IOException
- * @throws TException
- */
- String printGlobalPrivs() throws IOException, TException {
- PrincipalPrivilegeSet pps = getGlobalPrivs();
- if (pps == null) return "No global privileges";
- else return dumpThriftObject(pps);
- }
-
- /**********************************************************************************************
- * Partition related methods
- *********************************************************************************************/
-
- /**
- * Fetch one partition
- * @param dbName database table is in
- * @param tableName table partition is in
- * @param partVals list of values that specify the partition, given in the same order as the
- * columns they belong to
- * @return The partition objec,t or null if there is no such partition
- * @throws IOException
- */
- Partition getPartition(String dbName, String tableName, List<String> partVals)
- throws IOException {
- return getPartition(dbName, tableName, partVals, true);
- }
-
- /**
- * Get a set of specific partitions. This cannot be used to do a scan, each partition must be
- * completely specified. This does not use the partition cache.
- * @param dbName database table is in
- * @param tableName table partitions are in
- * @param partValLists list of list of values, each list should uniquely identify one partition
- * @return a list of partition objects.
- * @throws IOException
- */
- List<Partition> getPartitions(String dbName, String tableName, List<String> partTypes,
- List<List<String>> partValLists) throws IOException {
- List<Partition> parts = new ArrayList<>(partValLists.size());
- List<Get> gets = new ArrayList<>(partValLists.size());
- for (List<String> partVals : partValLists) {
- byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName, partTypes, partVals);
- Get get = new Get(key);
- get.addColumn(CATALOG_CF, CATALOG_COL);
- gets.add(get);
- }
- HTableInterface htab = conn.getHBaseTable(PART_TABLE);
- Result[] results = htab.get(gets);
- for (int i = 0; i < results.length; i++) {
- HBaseUtils.StorageDescriptorParts sdParts =
- HBaseUtils.deserializePartition(dbName, tableName, partValLists.get(i),
- results[i].getValue(CATALOG_CF, CATALOG_COL));
- StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
- HBaseUtils.assembleStorageDescriptor(sd, sdParts);
- parts.add(sdParts.containingPartition);
- }
-
- return parts;
- }
-
- /**
- * Add a partition. This should only be called for new partitions. For altering existing
- * partitions this should not be called as it will blindly increment the ref counter for the
- * storage descriptor.
- * @param partition partition object to add
- * @throws IOException
- */
- void putPartition(Partition partition) throws IOException {
- byte[] hash = putStorageDescriptor(partition.getSd());
- byte[][] serialized = HBaseUtils.serializePartition(partition,
- HBaseUtils.getPartitionKeyTypes(getTable(partition.getDbName(), partition.getTableName()).getPartitionKeys()), hash);
- store(PART_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- partCache.put(partition.getDbName(), partition.getTableName(), partition);
- }
-
- /**
- * Replace an existing partition.
- * @param oldPart partition to be replaced
- * @param newPart partitiion to replace it with
- * @throws IOException
- */
- void replacePartition(Partition oldPart, Partition newPart, List<String> partTypes) throws IOException {
- byte[] hash;
- byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldPart.getSd(), md);
- byte[] newHash = HBaseUtils.hashStorageDescriptor(newPart.getSd(), md);
- if (Arrays.equals(oldHash, newHash)) {
- hash = oldHash;
- } else {
- decrementStorageDescriptorRefCount(oldPart.getSd());
- hash = putStorageDescriptor(newPart.getSd());
- }
- byte[][] serialized = HBaseUtils.serializePartition(newPart,
- HBaseUtils.getPartitionKeyTypes(getTable(newPart.getDbName(), newPart.getTableName()).getPartitionKeys()), hash);
- store(PART_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- partCache.put(newPart.getDbName(), newPart.getTableName(), newPart);
- if (!oldPart.getTableName().equals(newPart.getTableName())) {
- deletePartition(oldPart.getDbName(), oldPart.getTableName(), partTypes, oldPart.getValues());
- }
- }
-
- /**
- * Add a group of partitions. This should only be used when all partitions are new. It
- * blindly increments the ref count on the storage descriptor.
- * @param partitions list of partitions to add
- * @throws IOException
- */
- void putPartitions(List<Partition> partitions) throws IOException {
- List<Put> puts = new ArrayList<>(partitions.size());
- for (Partition partition : partitions) {
- byte[] hash = putStorageDescriptor(partition.getSd());
- List<String> partTypes = HBaseUtils.getPartitionKeyTypes(
- getTable(partition.getDbName(), partition.getTableName()).getPartitionKeys());
- byte[][] serialized = HBaseUtils.serializePartition(partition, partTypes, hash);
- Put p = new Put(serialized[0]);
- p.add(CATALOG_CF, CATALOG_COL, serialized[1]);
- puts.add(p);
- partCache.put(partition.getDbName(), partition.getTableName(), partition);
- }
- HTableInterface htab = conn.getHBaseTable(PART_TABLE);
- htab.put(puts);
- conn.flush(htab);
- }
-
- void replacePartitions(List<Partition> oldParts, List<Partition> newParts, List<String> oldPartTypes) throws IOException {
- if (oldParts.size() != newParts.size()) {
- throw new RuntimeException("Number of old and new partitions must match.");
- }
- List<Put> puts = new ArrayList<>(newParts.size());
- for (int i = 0; i < newParts.size(); i++) {
- byte[] hash;
- byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldParts.get(i).getSd(), md);
- byte[] newHash = HBaseUtils.hashStorageDescriptor(newParts.get(i).getSd(), md);
- if (Arrays.equals(oldHash, newHash)) {
- hash = oldHash;
- } else {
- decrementStorageDescriptorRefCount(oldParts.get(i).getSd());
- hash = putStorageDescriptor(newParts.get(i).getSd());
- }
- Partition newPart = newParts.get(i);
- byte[][] serialized = HBaseUtils.serializePartition(newPart,
- HBaseUtils.getPartitionKeyTypes(getTable(newPart.getDbName(), newPart.getTableName()).getPartitionKeys()), hash);
- Put p = new Put(serialized[0]);
- p.add(CATALOG_CF, CATALOG_COL, serialized[1]);
- puts.add(p);
- partCache.put(newParts.get(i).getDbName(), newParts.get(i).getTableName(), newParts.get(i));
- if (!newParts.get(i).getTableName().equals(oldParts.get(i).getTableName())) {
- // We need to remove the old record as well.
- deletePartition(oldParts.get(i).getDbName(), oldParts.get(i).getTableName(), oldPartTypes,
- oldParts.get(i).getValues(), false);
- }
- }
- HTableInterface htab = conn.getHBaseTable(PART_TABLE);
- htab.put(puts);
- conn.flush(htab);
- }
-
- /**
- * Find all the partitions in a table.
- * @param dbName name of the database the table is in
- * @param tableName table name
- * @param maxPartitions max partitions to fetch. If negative all partitions will be returned.
- * @return List of partitions that match the criteria.
- * @throws IOException
- */
- List<Partition> scanPartitionsInTable(String dbName, String tableName, int maxPartitions)
- throws IOException {
- if (maxPartitions < 0) maxPartitions = Integer.MAX_VALUE;
- Collection<Partition> cached = partCache.getAllForTable(dbName, tableName);
- if (cached != null) {
- return maxPartitions < cached.size()
- ? new ArrayList<>(cached).subList(0, maxPartitions)
- : new ArrayList<>(cached);
- }
- byte[] keyPrefix = HBaseUtils.buildPartitionKey(dbName, tableName, new ArrayList<String>(),
- new ArrayList<String>(), false);
- List<Partition> parts = scanPartitionsWithFilter(dbName, tableName, keyPrefix,
- HBaseUtils.getEndPrefix(keyPrefix), -1, null);
- partCache.put(dbName, tableName, parts, true);
- return maxPartitions < parts.size() ? parts.subList(0, maxPartitions) : parts;
- }
-
- /**
- * Scan partitions based on partial key information.
- * @param dbName name of database, required
- * @param tableName name of table, required
- * @param partVals partial specification of values. Any values that are unknown can instead be
- * a '*'. For example, if a table had two partition columns date
- * and region (in that order), and partitions ('today', 'na'), ('today', 'eu'),
- * ('tomorrow', 'na'), ('tomorrow', 'eu') then passing ['today', '*'] would return
- * ('today', 'na') and ('today', 'eu') while passing ['*', 'eu'] would return
- * ('today', 'eu') and ('tomorrow', 'eu'). Also the list can terminate early,
- * which will be the equivalent of adding '*' for all non-included values.
- * I.e. ['today'] is the same as ['today', '*'].
- * @param maxPartitions Maximum number of entries to return.
- * @return list of partitions that match the specified information
- * @throws IOException
- * @throws org.apache.hadoop.hive.metastore.api.NoSuchObjectException if the table containing
- * the partitions can't be found.
- */
- List<Partition> scanPartitions(String dbName, String tableName, List<String> partVals,
- int maxPartitions) throws IOException, NoSuchObjectException {
-
- PartitionScanInfo psi = scanPartitionsInternal(dbName, tableName, partVals, maxPartitions);
- List<Partition> parts = scanPartitionsWithFilter(dbName, tableName, psi.keyPrefix,
- psi.endKeyPrefix, maxPartitions, psi.filter);
- partCache.put(dbName, tableName, parts, false);
- return parts;
- }
-
- List<Partition> scanPartitions(String dbName, String tableName, byte[] keyStart, byte[] keyEnd,
- Filter filter, int maxPartitions)
- throws IOException, NoSuchObjectException {
- byte[] startRow = keyStart;
- byte[] endRow;
- if (keyEnd == null || keyEnd.length == 0) {
- // stop when current db+table entries are over
- endRow = HBaseUtils.getEndPrefix(startRow);
- } else {
- endRow = keyEnd;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Scanning partitions with start row <" + new String(startRow) + "> and end row <"
- + new String(endRow) + ">");
- }
- return scanPartitionsWithFilter(dbName, tableName, startRow, endRow, maxPartitions, filter);
- }
-
- /**
- * Delete a partition
- * @param dbName database name that table is in
- * @param tableName table partition is in
- * @param partVals partition values that define this partition, in the same order as the
- * partition columns they are values for
- * @throws IOException
- */
- void deletePartition(String dbName, String tableName, List<String> partTypes,
- List<String> partVals) throws IOException {
- deletePartition(dbName, tableName, partTypes, partVals, true);
- }
-
- /**
- * Print out a partition.
- * @param partKey The key for the partition. This must include dbname.tablename._partkeys_
- * where _partkeys_ is a dot separated list of partition values in the proper
- * order.
- * @return string containing the partition
- * @throws IOException
- * @throws TException
- */
- String printPartition(String partKey) throws IOException, TException {
- // First figure out the table and fetch it
- String[] partKeyParts = partKey.split(HBaseUtils.KEY_SEPARATOR_STR);
- if (partKeyParts.length < 3) return noSuch(partKey, "partition");
- Table table = getTable(partKeyParts[0], partKeyParts[1]);
- if (table == null) return noSuch(partKey, "partition");
-
- byte[] key = HBaseUtils.buildPartitionKey(partKeyParts[0], partKeyParts[1],
- HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys()),
- Arrays.asList(Arrays.copyOfRange(partKeyParts, 2, partKeyParts.length)));
- @SuppressWarnings("deprecation")
- HTableInterface htab = conn.getHBaseTable(PART_TABLE);
- Get g = new Get(key);
- g.addColumn(CATALOG_CF, CATALOG_COL);
- g.addFamily(STATS_CF);
- Result result = htab.get(g);
- if (result.isEmpty()) return noSuch(partKey, "partition");
- return printOnePartition(result);
- }
-
- /**
- * Print partitions
- * @param partKey a partial partition key. This must match the beginings of the partition key.
- * It can be just dbname.tablename, or dbname.table.pval... where pval are the
- * partition values in order. They must be in the correct order and they must
- * be literal values (no regular expressions)
- * @return partitions as strings
- * @throws IOException
- * @throws TException
- */
- List<String> printPartitions(String partKey) throws IOException, TException {
- // First figure out the table and fetch it
- // Split on dot here rather than the standard separator because this will be passed in as a
- // regex, even though we aren't fully supporting regex's.
- String[] partKeyParts = partKey.split("\\.");
- if (partKeyParts.length < 2) return noMatch(partKey, "partition");
- List<String> partVals = partKeyParts.length == 2 ? Arrays.asList("*") :
- Arrays.asList(Arrays.copyOfRange(partKeyParts, 2, partKeyParts.length));
- PartitionScanInfo psi;
- try {
- psi =
- scanPartitionsInternal(partKeyParts[0], partKeyParts[1], partVals, -1);
- } catch (NoSuchObjectException e) {
- return noMatch(partKey, "partition");
- }
-
- @SuppressWarnings("deprecation")
- HTableInterface htab = conn.getHBaseTable(PART_TABLE);
- Scan scan = new Scan();
- scan.addColumn(CATALOG_CF, CATALOG_COL);
- scan.addFamily(STATS_CF);
- scan.setStartRow(psi.keyPrefix);
- scan.setStopRow(psi.endKeyPrefix);
- scan.setFilter(psi.filter);
- Iterator<Result> iter = htab.getScanner(scan).iterator();
- if (!iter.hasNext()) return noMatch(partKey, "partition");
- List<String> lines = new ArrayList<>();
- while (iter.hasNext()) {
- lines.add(printOnePartition(iter.next()));
- }
- return lines;
- }
-
- int getPartitionCount() throws IOException {
- Filter fil = new FirstKeyOnlyFilter();
- Iterator<Result> iter = scan(PART_TABLE, fil);
- return Iterators.size(iter);
- }
-
- private String printOnePartition(Result result) throws IOException, TException {
- byte[] key = result.getRow();
- HBaseUtils.StorageDescriptorParts sdParts =
- HBaseUtils.deserializePartition(key, result.getValue(CATALOG_CF, CATALOG_COL), this);
- StringBuilder builder = new StringBuilder();
- builder.append(dumpThriftObject(sdParts.containingPartition))
- .append(" sdHash: ")
- .append(Base64.encodeBase64URLSafeString(sdParts.sdHash))
- .append(" stats:");
- NavigableMap<byte[], byte[]> statsCols = result.getFamilyMap(STATS_CF);
- for (Map.Entry<byte[], byte[]> statsCol : statsCols.entrySet()) {
- builder.append(" column ")
- .append(new String(statsCol.getKey(), HBaseUtils.ENCODING))
- .append(": ");
- ColumnStatistics pcs = buildColStats(key, false);
- ColumnStatisticsObj cso = HBaseUtils.deserializeStatsForOneColumn(pcs, statsCol.getValue());
- builder.append(dumpThriftObject(cso));
- }
- return builder.toString();
- }
-
- private void deletePartition(String dbName, String tableName, List<String> partTypes,
- List<String> partVals, boolean decrementRefCnt) throws IOException {
- // Find the partition so I can get the storage descriptor and drop it
- partCache.remove(dbName, tableName, partVals);
- if (decrementRefCnt) {
- Partition p = getPartition(dbName, tableName, partVals, false);
- decrementStorageDescriptorRefCount(p.getSd());
- }
- byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName, partTypes, partVals);
- delete(PART_TABLE, key, null, null);
- }
-
- private Partition getPartition(String dbName, String tableName, List<String> partVals,
- boolean populateCache) throws IOException {
- Partition cached = partCache.get(dbName, tableName, partVals);
- if (cached != null) return cached;
- byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName,
- HBaseUtils.getPartitionKeyTypes(getTable(dbName, tableName).getPartitionKeys()), partVals);
- byte[] serialized = read(PART_TABLE, key, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return null;
- HBaseUtils.StorageDescriptorParts sdParts =
- HBaseUtils.deserializePartition(dbName, tableName, partVals, serialized);
- StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
- HBaseUtils.assembleStorageDescriptor(sd, sdParts);
- if (populateCache) partCache.put(dbName, tableName, sdParts.containingPartition);
- return sdParts.containingPartition;
- }
-
-
- private static class PartitionScanInfo {
- final String dbName;
- final String tableName;
- final byte[] keyPrefix;
- final byte[] endKeyPrefix;
- final int maxPartitions;
- final Filter filter;
-
- PartitionScanInfo(String d, String t, byte[] k, byte[] e, int m, Filter f) {
- dbName = d;
- tableName = t;
- keyPrefix = k;
- endKeyPrefix = e;
- maxPartitions = m;
- filter = f;
- }
-
- @Override
- public String toString() {
- return new StringBuilder("dbName:")
- .append(dbName)
- .append(" tableName:")
- .append(tableName)
- .append(" keyPrefix:")
- .append(Base64.encodeBase64URLSafeString(keyPrefix))
- .append(" endKeyPrefix:")
- .append(Base64.encodeBase64URLSafeString(endKeyPrefix))
- .append(" maxPartitions:")
- .append(maxPartitions)
- .append(" filter:")
- .append(filter.toString())
- .toString();
- }
- }
-
- private PartitionScanInfo scanPartitionsInternal(String dbName, String tableName,
- List<String> partVals, int maxPartitions)
- throws IOException, NoSuchObjectException {
- // First, build as much of the key as we can so that we make the scan as tight as possible.
- List<String> keyElements = new ArrayList<>();
- keyElements.add(dbName);
- keyElements.add(tableName);
-
- int firstStar = -1;
- for (int i = 0; i < partVals.size(); i++) {
- if ("*".equals(partVals.get(i))) {
- firstStar = i;
- break;
- } else {
- // empty string equals to null partition,
- // means star
- if (partVals.get(i).equals("")) {
- break;
- } else {
- keyElements.add(partVals.get(i));
- }
- }
- }
-
- byte[] keyPrefix;
- // We need to fetch the table to determine if the user fully specified the partitions or
- // not, as it affects how we build the key.
- Table table = getTable(dbName, tableName);
- if (table == null) {
- throw new NoSuchObjectException("Unable to find table " + dbName + "." + tableName);
- }
- keyPrefix = HBaseUtils.buildPartitionKey(dbName, tableName,
- HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys().subList(0, keyElements.size()-2)),
- keyElements.subList(2, keyElements.size()));
-
- // Now, build a filter out of the remaining keys
- List<PartitionKeyComparator.Range> ranges = new ArrayList<PartitionKeyComparator.Range>();
- List<Operator> ops = new ArrayList<Operator>();
- if (!(partVals.size() == table.getPartitionKeys().size() && firstStar == -1)) {
-
- for (int i = Math.max(0, firstStar);
- i < table.getPartitionKeys().size() && i < partVals.size(); i++) {
-
- if ("*".equals(partVals.get(i))) {
- PartitionKeyComparator.Operator op = new PartitionKeyComparator.Operator(
- PartitionKeyComparator.Operator.Type.LIKE,
- table.getPartitionKeys().get(i).getName(),
- ".*");
- ops.add(op);
- } else {
- PartitionKeyComparator.Range range = new PartitionKeyComparator.Range(
- table.getPartitionKeys().get(i).getName(),
- new PartitionKeyComparator.Mark(partVals.get(i), true),
- new PartitionKeyComparator.Mark(partVals.get(i), true));
- ranges.add(range);
- }
- }
- }
-
- Filter filter = null;
- if (!ranges.isEmpty() || !ops.isEmpty()) {
- filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new PartitionKeyComparator(
- StringUtils.join(HBaseUtils.getPartitionNames(table.getPartitionKeys()), ","),
- StringUtils.join(HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys()), ","),
- ranges, ops));
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Scanning partitions with prefix <" + new String(keyPrefix) + "> and filter <" +
- filter + ">");
- }
-
- return new PartitionScanInfo(dbName, tableName, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix),
- maxPartitions, filter);
- }
-
- private List<Partition> scanPartitionsWithFilter(String dbName, String tableName,
- byte[] startRow, byte [] endRow, int maxResults,
- Filter filter) throws IOException {
- Iterator<Result> iter =
- scan(PART_TABLE, startRow, endRow, CATALOG_CF, CATALOG_COL, filter);
- List<FieldSchema> tablePartitions = getTable(dbName, tableName).getPartitionKeys();
- List<Partition> parts = new ArrayList<>();
- int numToFetch = maxResults < 0 ? Integer.MAX_VALUE : maxResults;
- for (int i = 0; i < numToFetch && iter.hasNext(); i++) {
- Result result = iter.next();
- HBaseUtils.StorageDescriptorParts sdParts = HBaseUtils.deserializePartition(dbName, tableName,
- tablePartitions, result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL), conf);
- StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
- HBaseUtils.assembleStorageDescriptor(sd, sdParts);
- parts.add(sdParts.containingPartition);
- }
- return parts;
- }
-
- /**********************************************************************************************
- * Role related methods
- *********************************************************************************************/
-
- /**
- * Fetch the list of all roles for a user
- * @param userName name of the user
- * @return the list of all roles this user participates in
- * @throws IOException
- */
- List<String> getUserRoles(String userName) throws IOException {
- byte[] key = HBaseUtils.buildKey(userName);
- byte[] serialized = read(USER_TO_ROLE_TABLE, key, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return null;
- return HBaseUtils.deserializeRoleList(serialized);
- }
-
- /**
- * Find all roles directly participated in by a given principal. This builds the role cache
- * because it assumes that subsequent calls may be made to find roles participated in indirectly.
- * @param name username or role name
- * @param type user or role
- * @return map of role name to grant info for all roles directly participated in.
- */
- List<Role> getPrincipalDirectRoles(String name, PrincipalType type)
- throws IOException {
- buildRoleCache();
-
- Set<String> rolesFound = new HashSet<>();
- for (Map.Entry<String, HbaseMetastoreProto.RoleGrantInfoList> e : roleCache.entrySet()) {
- for (HbaseMetastoreProto.RoleGrantInfo giw : e.getValue().getGrantInfoList()) {
- if (HBaseUtils.convertPrincipalTypes(giw.getPrincipalType()) == type &&
- giw.getPrincipalName().equals(name)) {
- rolesFound.add(e.getKey());
- break;
- }
- }
- }
- List<Role> directRoles = new ArrayList<>(rolesFound.size());
- List<Get> gets = new ArrayList<>();
- HTableInterface htab = conn.getHBaseTable(ROLE_TABLE);
- for (String roleFound : rolesFound) {
- byte[] key = HBaseUtils.buildKey(roleFound);
- Get g = new Get(key);
- g.addColumn(CATALOG_CF, CATALOG_COL);
- gets.add(g);
- }
-
- Result[] results = htab.get(gets);
- for (int i = 0; i < results.length; i++) {
- byte[] serialized = results[i].getValue(CATALOG_CF, CATALOG_COL);
- if (serialized != null) {
- directRoles.add(HBaseUtils.deserializeRole(results[i].getRow(), serialized));
- }
- }
-
- return directRoles;
- }
-
- /**
- * Fetch all roles and users included directly in a given role.
- * @param roleName name of the principal
- * @return a list of all roles included in this role
- * @throws IOException
- */
- HbaseMetastoreProto.RoleGrantInfoList getRolePrincipals(String roleName)
- throws IOException, NoSuchObjectException {
- HbaseMetastoreProto.RoleGrantInfoList rolePrincipals = roleCache.get(roleName);
- if (rolePrincipals != null) return rolePrincipals;
- byte[] key = HBaseUtils.buildKey(roleName);
- byte[] serialized = read(ROLE_TABLE, key, CATALOG_CF, ROLES_COL);
- if (serialized == null) return null;
- rolePrincipals = HbaseMetastoreProto.RoleGrantInfoList.parseFrom(serialized);
- roleCache.put(roleName, rolePrincipals);
- return rolePrincipals;
- }
-
- /**
- * Given a role, find all users who are either directly or indirectly participate in this role.
- * This is expensive, it should be used sparingly. It scan the entire userToRole table and
- * does a linear search on each entry.
- * @param roleName name of the role
- * @return set of all users in the role
- * @throws IOException
- */
- Set<String> findAllUsersInRole(String roleName) throws IOException {
- // Walk the userToRole table and collect every user that matches this role.
- Set<String> users = new HashSet<>();
- Iterator<Result> iter = scan(USER_TO_ROLE_TABLE, CATALOG_CF, CATALOG_COL);
- while (iter.hasNext()) {
- Result result = iter.next();
- List<String> roleList =
- HBaseUtils.deserializeRoleList(result.getValue(CATALOG_CF, CATALOG_COL));
- for (String rn : roleList) {
- if (rn.equals(roleName)) {
- users.add(new String(result.getRow(), HBaseUtils.ENCODING));
- break;
- }
- }
- }
- return users;
- }
-
- /**
- * Add a principal to a role.
- * @param roleName name of the role to add principal to
- * @param grantInfo grant information for this principal.
- * @throws java.io.IOException
- * @throws NoSuchObjectException
- *
- */
- void addPrincipalToRole(String roleName, HbaseMetastoreProto.RoleGrantInfo grantInfo)
- throws IOException, NoSuchObjectException {
- HbaseMetastoreProto.RoleGrantInfoList proto = getRolePrincipals(roleName);
- List<HbaseMetastoreProto.RoleGrantInfo> rolePrincipals = new ArrayList<>();
- if (proto != null) {
- rolePrincipals.addAll(proto.getGrantInfoList());
- }
-
- rolePrincipals.add(grantInfo);
- proto = HbaseMetastoreProto.RoleGrantInfoList.newBuilder()
- .addAllGrantInfo(rolePrincipals)
- .build();
- byte[] key = HBaseUtils.buildKey(roleName);
- store(ROLE_TABLE, key, CATALOG_CF, ROLES_COL, proto.toByteArray());
- roleCache.put(roleName, proto);
- }
-
- /**
- * Drop a principal from a role.
- * @param roleName Name of the role to drop the principal from
- * @param principalName name of the principal to drop from the role
- * @param type user or role
- * @param grantOnly if this is true, just remove the grant option, don't actually remove the
- * user from the role.
- * @throws NoSuchObjectException
- * @throws IOException
- */
- void dropPrincipalFromRole(String roleName, String principalName, PrincipalType type,
- boolean grantOnly)
- throws NoSuchObjectException, IOException {
- HbaseMetastoreProto.RoleGrantInfoList proto = getRolePrincipals(roleName);
- if (proto == null) return;
- List<HbaseMetastoreProto.RoleGrantInfo> rolePrincipals = new ArrayList<>();
- rolePrincipals.addAll(proto.getGrantInfoList());
-
- for (int i = 0; i < rolePrincipals.size(); i++) {
- if (HBaseUtils.convertPrincipalTypes(rolePrincipals.get(i).getPrincipalType()) == type &&
- rolePrincipals.get(i).getPrincipalName().equals(principalName)) {
- if (grantOnly) {
- rolePrincipals.set(i,
- HbaseMetastoreProto.RoleGrantInfo.newBuilder(rolePrincipals.get(i))
- .setGrantOption(false)
- .build());
- } else {
- rolePrincipals.remove(i);
- }
- break;
- }
- }
- byte[] key = HBaseUtils.buildKey(roleName);
- proto = HbaseMetastoreProto.RoleGrantInfoList.newBuilder()
- .addAllGrantInfo(rolePrincipals)
- .build();
- store(ROLE_TABLE, key, CATALOG_CF, ROLES_COL, proto.toByteArray());
- roleCache.put(roleName, proto);
- }
-
- /**
- * Rebuild the row for a given user in the USER_TO_ROLE table. This is expensive. It
- * should be called as infrequently as possible.
- * @param userName name of the user
- * @throws IOException
- */
- void buildRoleMapForUser(String userName) throws IOException, NoSuchObjectException {
- // This is mega ugly. Hopefully we don't have to do this too often.
- // First, scan the role table and put it all in memory
- buildRoleCache();
- LOG.debug("Building role map for " + userName);
-
- // Second, find every role the user participates in directly.
- Set<String> rolesToAdd = new HashSet<>();
- Set<String> rolesToCheckNext = new HashSet<>();
- for (Map.Entry<String, HbaseMetastoreProto.RoleGrantInfoList> e : roleCache.entrySet()) {
- for (HbaseMetastoreProto.RoleGrantInfo grantInfo : e.getValue().getGrantInfoList()) {
- if (HBaseUtils.convertPrincipalTypes(grantInfo.getPrincipalType()) == PrincipalType.USER &&
- userName .equals(grantInfo.getPrincipalName())) {
- rolesToAdd.add(e.getKey());
- rolesToCheckNext.add(e.getKey());
- LOG.debug("Adding " + e.getKey() + " to list of roles user is in directly");
- break;
- }
- }
- }
-
- // Third, find every role the user participates in indirectly (that is, they have been
- // granted into role X and role Y has been granted into role X).
- while (rolesToCheckNext.size() > 0) {
- Set<String> tmpRolesToCheckNext = new HashSet<>();
- for (String roleName : rolesToCheckNext) {
- HbaseMetastoreProto.RoleGrantInfoList grantInfos = roleCache.get(roleName);
- if (grantInfos == null) continue; // happens when a role contains no grants
- for (HbaseMetastoreProto.RoleGrantInfo grantInfo : grantInfos.getGrantInfoList()) {
- if (HBaseUtils.convertPrincipalTypes(grantInfo.getPrincipalType()) == PrincipalType.ROLE &&
- rolesToAdd.add(grantInfo.getPrincipalName())) {
- tmpRolesToCheckNext.add(grantInfo.getPrincipalName());
- LOG.debug("Adding " + grantInfo.getPrincipalName() +
- " to list of roles user is in indirectly");
- }
- }
- }
- rolesToCheckNext = tmpRolesToCheckNext;
- }
-
- byte[] key = HBaseUtils.buildKey(userName);
- byte[] serialized = HBaseUtils.serializeRoleList(new ArrayList<>(rolesToAdd));
- store(USER_TO_ROLE_TABLE, key, CATALOG_CF, CATALOG_COL, serialized);
- }
-
- /**
- * Remove all of the grants for a role. This is not cheap.
- * @param roleName Role to remove from all other roles and grants
- * @throws IOException
- */
- void removeRoleGrants(String roleName) throws IOException {
- buildRoleCache();
-
- List<Put> puts = new ArrayList<>();
- // First, walk the role table and remove any references to this role
- for (Map.Entry<String, HbaseMetastoreProto.RoleGrantInfoList> e : roleCache.entrySet()) {
- boolean madeAChange = false;
- List<HbaseMetastoreProto.RoleGrantInfo> rgil = new ArrayList<>();
- rgil.addAll(e.getValue().getGrantInfoList());
- for (int i = 0; i < rgil.size(); i++) {
- if (HBaseUtils.convertPrincipalTypes(rgil.get(i).getPrincipalType()) == PrincipalType.ROLE &&
- rgil.get(i).getPrincipalName().equals(roleName)) {
- rgil.remove(i);
- madeAChange = true;
- break;
- }
- }
- if (madeAChange) {
- Put put = new Put(HBaseUtils.buildKey(e.getKey()));
- HbaseMetastoreProto.RoleGrantInfoList proto =
- HbaseMetastoreProto.RoleGrantInfoList.newBuilder()
- .addAllGrantInfo(rgil)
- .build();
- put.add(CATALOG_CF, ROLES_COL, proto.toByteArray());
- puts.add(put);
- roleCache.put(e.getKey(), proto);
- }
- }
-
- if (puts.size() > 0) {
- HTableInterface htab = conn.getHBaseTable(ROLE_TABLE);
- htab.put(puts);
- conn.flush(htab);
- }
-
- // Remove any global privileges held by this role
- PrincipalPrivilegeSet global = getGlobalPrivs();
- if (global != null &&
- global.getRolePrivileges() != null &&
- global.getRolePrivileges().remove(roleName) != null) {
- putGlobalPrivs(global);
- }
-
- // Now, walk the db table
- puts.clear();
- List<Database> dbs = scanDatabases(null);
- if (dbs == null) dbs = new ArrayList<>(); // rare, but can happen
- for (Database db : dbs) {
- if (db.getPrivileges() != null &&
- db.getPrivileges().getRolePrivileges() != null &&
- db.getPrivileges().getRolePrivileges().remove(roleName) != null) {
- byte[][] serialized = HBaseUtils.serializeDatabase(db);
- Put put = new Put(serialized[0]);
- put.add(CATALOG_CF, CATALOG_COL, serialized[1]);
- puts.add(put);
- }
- }
-
- if (puts.size() > 0) {
- HTableInterface htab = conn.getHBaseTable(DB_TABLE);
- htab.put(puts);
- conn.flush(htab);
- }
-
- // Finally, walk the table table
- puts.clear();
- for (Database db : dbs) {
- List<Table> tables = scanTables(db.getName(), null);
- if (tables != null) {
- for (Table table : tables) {
- if (table.getPrivileges() != null &&
- table.getPrivileges().getRolePrivileges() != null &&
- table.getPrivileges().getRolePrivileges().remove(roleName) != null) {
- byte[][] serialized = HBaseUtils.serializeTable(table,
- HBaseUtils.hashStorageDescriptor(table.getSd(), md));
- Put put = new Put(serialized[0]);
- put.add(CATALOG_CF, CATALOG_COL, serialized[1]);
- puts.add(put);
- }
- }
- }
- }
-
- if (puts.size() > 0) {
- HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
- htab.put(puts);
- conn.flush(htab);
- }
- }
-
- /**
- * Fetch a role
- * @param roleName name of the role
- * @return role object, or null if no such role
- * @throws IOException
- */
- Role getRole(String roleName) throws IOException {
- byte[] key = HBaseUtils.buildKey(roleName);
- byte[] serialized = read(ROLE_TABLE, key, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return null;
- return HBaseUtils.deserializeRole(roleName, serialized);
- }
-
- /**
- * Get a list of roles.
- * @return list of all known roles.
- * @throws IOException
- */
- List<Role> scanRoles() throws IOException {
- return scanRoles(null);
- }
-
- /**
- * Add a new role
- * @param role role object
- * @throws IOException
- */
- void putRole(Role role) throws IOException {
- byte[][] serialized = HBaseUtils.serializeRole(role);
- store(ROLE_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- }
-
- /**
- * Drop a role
- * @param roleName name of role to drop
- * @throws IOException
- */
- void deleteRole(String roleName) throws IOException {
- byte[] key = HBaseUtils.buildKey(roleName);
- delete(ROLE_TABLE, key, null, null);
- roleCache.remove(roleName);
- }
-
- String printRolesForUser(String userName) throws IOException {
- List<String> roles = getUserRoles(userName);
- if (roles == null || roles.size() == 0) return noSuch(userName, "user");
- return org.apache.commons.lang.StringUtils.join(roles, ',');
- }
-
- List<String> printRolesForUsers(String regex) throws IOException {
- Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
- Iterator<Result> iter = scan(USER_TO_ROLE_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
- List<String> lines = new ArrayList<>();
- while (iter.hasNext()) {
- Result result = iter.next();
- lines.add(new String(result.getRow(), HBaseUtils.ENCODING) + ": " +
- org.apache.commons.lang.StringUtils.join(
- HBaseUtils.deserializeRoleList(result.getValue(CATALOG_CF, CATALOG_COL)), ','));
- }
- if (lines.size() == 0) lines = noMatch(regex, "user");
- return lines;
- }
-
- /**
- * Print out a role
- * @param name name of role to print
- * @return string printout of role
- */
- String printRole(String name) throws IOException, TException {
- Role role = getRole(name);
- if (role == null) return noSuch(name, "role");
- else return dumpThriftObject(role);
- }
-
- /**
- * Print out roles.
- * @param regex regular to use to search for roles
- * @return string printout of roles
- * @throws IOException
- * @throws TException
- */
- List<String> printRoles(String regex) throws IOException, TException {
- List<Role> roles = scanRoles(regex);
- if (roles.size() == 0) {
- return noMatch(regex, "role");
- } else {
- List<String> lines = new ArrayList<>();
- for (Role role : roles) lines.add(dumpThriftObject(role));
- return lines;
- }
- }
-
- private List<Role> scanRoles(String regex) throws IOException {
- Filter filter = null;
- if (regex != null) {
- filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
- }
- Iterator<Result> iter = scan(ROLE_TABLE, null, null, CATALOG_CF, CATALOG_COL, filter);
- List<Role> roles = new ArrayList<>();
- while (iter.hasNext()) {
- Result result = iter.next();
- roles.add(HBaseUtils.deserializeRole(result.getRow(),
- result.getValue(CATALOG_CF, CATALOG_COL)));
- }
- return roles;
- }
-
- private void buildRoleCache() throws IOException {
- if (!entireRoleTableInCache) {
- Iterator<Result> roles = scan(ROLE_TABLE, CATALOG_CF, ROLES_COL);
- while (roles.hasNext()) {
- Result res = roles.next();
- String roleName = new String(res.getRow(), HBaseUtils.ENCODING);
- HbaseMetastoreProto.RoleGrantInfoList grantInfos =
- HbaseMetastoreProto.RoleGrantInfoList.parseFrom(res.getValue(CATALOG_CF, ROLES_COL));
- roleCache.put(roleName, grantInfos);
- }
- entireRoleTableInCache = true;
- }
- }
-
- /**********************************************************************************************
- * Table related methods
- *********************************************************************************************/
-
- /**
- * Fetch a table object
- * @param dbName database the table is in
- * @param tableName table name
- * @return Table object, or null if no such table
- * @throws IOException
- */
- Table getTable(String dbName, String tableName) throws IOException {
- return getTable(dbName, tableName, true);
- }
-
- /**
- * Fetch a list of table objects.
- * @param dbName Database that all fetched tables are in
- * @param tableNames list of table names
- * @return list of tables, in the same order as the provided names.
- * @throws IOException
- */
- List<Table> getTables(String dbName, List<String> tableNames) throws IOException {
- // I could implement getTable in terms of this method. But it is such a core function
- // that I don't want to slow it down for the much less common fetching of multiple tables.
- List<Table> results = new ArrayList<>(tableNames.size());
- ObjectPair<String, String>[] hashKeys = new ObjectPair[tableNames.size()];
- boolean atLeastOneMissing = false;
- for (int i = 0; i < tableNames.size(); i++) {
- hashKeys[i] = new ObjectPair<>(dbName, tableNames.get(i));
- // The result may be null, but we still want to add it so that we have a slot in the list
- // for it.
- results.add(tableCache.get(hashKeys[i]));
- if (results.get(i) == null) atLeastOneMissing = true;
- }
- if (!atLeastOneMissing) return results;
-
- // Now build a single get that will fetch the remaining tables
- List<Get> gets = new ArrayList<>();
- HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
- for (int i = 0; i < tableNames.size(); i++) {
- if (results.get(i) != null) continue;
- byte[] key = HBaseUtils.buildKey(dbName, tableNames.get(i));
- Get g = new Get(key);
- g.addColumn(CATALOG_CF, CATALOG_COL);
- gets.add(g);
- }
- Result[] res = htab.get(gets);
- for (int i = 0, nextGet = 0; i < tableNames.size(); i++) {
- if (results.get(i) != null) continue;
- byte[] serialized = res[nextGet++].getValue(CATALOG_CF, CATALOG_COL);
- if (serialized != null) {
- HBaseUtils.StorageDescriptorParts sdParts =
- HBaseUtils.deserializeTable(dbName, tableNames.get(i), serialized);
- StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
- HBaseUtils.assembleStorageDescriptor(sd, sdParts);
- tableCache.put(hashKeys[i], sdParts.containingTable);
- results.set(i, sdParts.containingTable);
- }
- }
- return results;
- }
-
- /**
- * Get a list of tables.
- * @param dbName Database these tables are in
- * @param regex Regular expression to use in searching for table names. It is expected to
- * be a Java regular expression. If it is null then all tables in the indicated
- * database will be returned.
- * @return list of tables matching the regular expression.
- * @throws IOException
- */
- List<Table> scanTables(String dbName, String regex) throws IOException {
- // There's no way to know whether all the tables we are looking for are
- // in the cache, so we would need to scan one way or another. Thus there's no value in hitting
- // the cache for this function.
- byte[] keyPrefix = null;
- if (dbName != null) {
- keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName);
- }
- Filter filter = null;
- if (regex != null) {
- filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
- }
- Iterator<Result> iter =
- scan(TABLE_TABLE, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix),
- CATALOG_CF, CATALOG_COL, filter);
- List<Table> tables = new ArrayList<>();
- while (iter.hasNext()) {
- Result result = iter.next();
- HBaseUtils.StorageDescriptorParts sdParts =
- HBaseUtils.deserializeTable(result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL));
- StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
- HBaseUtils.assembleStorageDescriptor(sd, sdParts);
- tables.add(sdParts.containingTable);
- }
- return tables;
- }
-
- /**
- * Put a table object. This should only be called when the table is new (create table) as it
- * will blindly add/increment the storage descriptor. If you are altering an existing table
- * call {@link #replaceTable} instead.
- * @param table table object
- * @throws IOException
- */
- void putTable(Table table) throws IOException {
- byte[] hash = putStorageDescriptor(table.getSd());
- byte[][] serialized = HBaseUtils.serializeTable(table, hash);
- store(TABLE_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- tableCache.put(new ObjectPair<>(table.getDbName(), table.getTableName()), table);
- }
-
- /**
- * Replace an existing table. This will also compare the storage descriptors and see if the
- * reference count needs to be adjusted
- * @param oldTable old version of the table
- * @param newTable new version of the table
- */
- void replaceTable(Table oldTable, Table newTable) throws IOException {
- byte[] hash;
- byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldTable.getSd(), md);
- byte[] newHash = HBaseUtils.hashStorageDescriptor(newTable.getSd(), md);
- if (Arrays.equals(oldHash, newHash)) {
- hash = oldHash;
- } else {
- decrementStorageDescriptorRefCount(oldTable.getSd());
- hash = putStorageDescriptor(newTable.getSd());
- }
- byte[][] serialized = HBaseUtils.serializeTable(newTable, hash);
- store(TABLE_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- tableCache.put(new ObjectPair<>(newTable.getDbName(), newTable.getTableName()), newTable);
- if (!oldTable.getTableName().equals(newTable.getTableName())) {
- deleteTable(oldTable.getDbName(), oldTable.getTableName());
- }
- }
-
- /**
- * Delete a table
- * @param dbName name of database table is in
- * @param tableName table to drop
- * @throws IOException
- */
- void deleteTable(String dbName, String tableName) throws IOException {
- deleteTable(dbName, tableName, true);
- }
-
- /**
- * Print out a table.
- * @param name The name for the table. This must include dbname.tablename
- * @return string containing the table
- * @throws IOException
- * @throws TException
- */
- String printTable(String name) throws IOException, TException {
- byte[] key = HBaseUtils.buildKey(name);
- @SuppressWarnings("deprecation")
- HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
- Get g = new Get(key);
- g.addColumn(CATALOG_CF, CATALOG_COL);
- g.addFamily(STATS_CF);
- Result result = htab.get(g);
- if (result.isEmpty()) return noSuch(name, "table");
- return printOneTable(result);
- }
-
- /**
- * Print tables
- * @param regex to use to find the tables. Remember that dbname is in each
- * table name.
- * @return tables as strings
- * @throws IOException
- * @throws TException
- */
- List<String> printTables(String regex) throws IOException, TException {
- Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
- @SuppressWarnings("deprecation")
- HTableInterface htab = conn.getHBaseTable(TABLE_TABLE);
- Scan scan = new Scan();
- scan.addColumn(CATALOG_CF, CATALOG_COL);
- scan.addFamily(STATS_CF);
- scan.setFilter(filter);
- Iterator<Result> iter = htab.getScanner(scan).iterator();
- if (!iter.hasNext()) return noMatch(regex, "table");
- List<String> lines = new ArrayList<>();
- while (iter.hasNext()) {
- lines.add(printOneTable(iter.next()));
- }
- return lines;
- }
-
- int getTableCount() throws IOException {
- Filter fil = new FirstKeyOnlyFilter();
- Iterator<Result> iter = scan(TABLE_TABLE, fil);
- return Iterators.size(iter);
- }
-
- private String printOneTable(Result result) throws IOException, TException {
- byte[] key = result.getRow();
- HBaseUtils.StorageDescriptorParts sdParts =
- HBaseUtils.deserializeTable(key, result.getValue(CATALOG_CF, CATALOG_COL));
- StringBuilder builder = new StringBuilder();
- builder.append(dumpThriftObject(sdParts.containingTable))
- .append(" sdHash: ")
- .append(Base64.encodeBase64URLSafeString(sdParts.sdHash))
- .append(" stats:");
- NavigableMap<byte[], byte[]> statsCols = result.getFamilyMap(STATS_CF);
- for (Map.Entry<byte[], byte[]> statsCol : statsCols.entrySet()) {
- builder.append(" column ")
- .append(new String(statsCol.getKey(), HBaseUtils.ENCODING))
- .append(": ");
- ColumnStatistics pcs = buildColStats(key, true);
- ColumnStatisticsObj cso = HBaseUtils.deserializeStatsForOneColumn(pcs, statsCol.getValue());
- builder.append(dumpThriftObject(cso));
- }
- // Add the primary key
- List<SQLPrimaryKey> pk = getPrimaryKey(sdParts.containingTable.getDbName(),
- sdParts.containingTable.getTableName());
- if (pk != null && pk.size() > 0) {
- builder.append(" primary key: ");
- for (SQLPrimaryKey pkcol : pk) builder.append(dumpThriftObject(pkcol));
- }
-
- // Add any foreign keys
- List<SQLForeignKey> fks = getForeignKeys(sdParts.containingTable.getDbName(),
- sdParts.containingTable.getTableName());
- if (fks != null && fks.size() > 0) {
- builder.append(" foreign keys: ");
- for (SQLForeignKey fkcol : fks) builder.append(dumpThriftObject(fkcol));
-
- }
- return builder.toString();
- }
-
- private void deleteTable(String dbName, String tableName, boolean decrementRefCnt)
- throws IOException {
- tableCache.remove(new ObjectPair<>(dbName, tableName));
- if (decrementRefCnt) {
- // Find the table so I can get the storage descriptor and drop it
- Table t = getTable(dbName, tableName, false);
- decrementStorageDescriptorRefCount(t.getSd());
- }
- byte[] key = HBaseUtils.buildKey(dbName, tableName);
- delete(TABLE_TABLE, key, null, null);
- }
-
- private Table getTable(String dbName, String tableName, boolean populateCache)
- throws IOException {
- ObjectPair<String, String> hashKey = new ObjectPair<>(dbName, tableName);
- Table cached = tableCache.get(hashKey);
- if (cached != null) return cached;
- byte[] key = HBaseUtils.buildKey(dbName, tableName);
- byte[] serialized = read(TABLE_TABLE, key, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return null;
- HBaseUtils.StorageDescriptorParts sdParts =
- HBaseUtils.deserializeTable(dbName, tableName, serialized);
- StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
- HBaseUtils.assembleStorageDescriptor(sd, sdParts);
- if (populateCache) tableCache.put(hashKey, sdParts.containingTable);
- return sdParts.containingTable;
- }
-
- /**********************************************************************************************
- * Index related methods
- *********************************************************************************************/
-
- /**
- * Put an index object. This should only be called when the index is new (create index) as it
- * will blindly add/increment the storage descriptor. If you are altering an existing index
- * call {@link #replaceIndex} instead.
- * @param index index object
- * @throws IOException
- */
- void putIndex(Index index) throws IOException {
- byte[] hash = putStorageDescriptor(index.getSd());
- byte[][] serialized = HBaseUtils.serializeIndex(index, hash);
- store(INDEX_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- }
-
- /**
- * Fetch an index object
- * @param dbName database the table is in
- * @param origTableName original table name
- * @param indexName index name
- * @return Index object, or null if no such table
- * @throws IOException
- */
- Index getIndex(String dbName, String origTableName, String indexName) throws IOException {
- byte[] key = HBaseUtils.buildKey(dbName, origTableName, indexName);
- byte[] serialized = read(INDEX_TABLE, key, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return null;
- HBaseUtils.StorageDescriptorParts sdParts =
- HBaseUtils.deserializeIndex(dbName, origTableName, indexName, serialized);
- StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
- HBaseUtils.assembleStorageDescriptor(sd, sdParts);
- return sdParts.containingIndex;
- }
-
- /**
- * Delete a table
- * @param dbName name of database table is in
- * @param origTableName table the index is built on
- * @param indexName index name
- * @throws IOException
- */
- void deleteIndex(String dbName, String origTableName, String indexName) throws IOException {
- deleteIndex(dbName, origTableName, indexName, true);
- }
-
- void deleteIndex(String dbName, String origTableName, String indexName, boolean decrementRefCnt)
- throws IOException {
- // Find the index so I can get the storage descriptor and drop it
- if (decrementRefCnt) {
- Index index = getIndex(dbName, origTableName, indexName);
- decrementStorageDescriptorRefCount(index.getSd());
- }
- byte[] key = HBaseUtils.buildKey(dbName, origTableName, indexName);
- delete(INDEX_TABLE, key, null, null);
- }
-
- /**
- * Get a list of tables.
- * @param dbName Database these tables are in
- * @param origTableName original table name
- * @param maxResults max indexes to fetch. If negative all indexes will be returned.
- * @return list of indexes of the table
- * @throws IOException
- */
- List<Index> scanIndexes(String dbName, String origTableName, int maxResults) throws IOException {
- // There's no way to know whether all the tables we are looking for are
- // in the cache, so we would need to scan one way or another. Thus there's no value in hitting
- // the cache for this function.
- byte[] keyPrefix = null;
- if (dbName != null) {
- keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName, origTableName);
- }
- Iterator<Result> iter = scan(INDEX_TABLE, keyPrefix, HBaseUtils.getEndPrefix(keyPrefix),
- CATALOG_CF, CATALOG_COL, null);
- List<Index> indexes = new ArrayList<>();
- int numToFetch = maxResults < 0 ? Integer.MAX_VALUE : maxResults;
- for (int i = 0; i < numToFetch && iter.hasNext(); i++) {
- Result result = iter.next();
- HBaseUtils.StorageDescriptorParts sdParts = HBaseUtils.deserializeIndex(result.getRow(),
- result.getValue(CATALOG_CF, CATALOG_COL));
- StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
- HBaseUtils.assembleStorageDescriptor(sd, sdParts);
- indexes.add(sdParts.containingIndex);
- }
- return indexes;
- }
-
- /**
- * Replace an existing index. This will also compare the storage descriptors and see if the
- * reference count needs to be adjusted
- * @param oldIndex old version of the index
- * @param newIndex new version of the index
- */
- void replaceIndex(Index oldIndex, Index newIndex) throws IOException {
- byte[] hash;
- byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldIndex.getSd(), md);
- byte[] newHash = HBaseUtils.hashStorageDescriptor(newIndex.getSd(), md);
- if (Arrays.equals(oldHash, newHash)) {
- hash = oldHash;
- } else {
- decrementStorageDescriptorRefCount(oldIndex.getSd());
- hash = putStorageDescriptor(newIndex.getSd());
- }
- byte[][] serialized = HBaseUtils.serializeIndex(newIndex, hash);
- store(INDEX_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- if (!(oldIndex.getDbName().equals(newIndex.getDbName()) &&
- oldIndex.getOrigTableName().equals(newIndex.getOrigTableName()) &&
- oldIndex.getIndexName().equals(newIndex.getIndexName()))) {
- deleteIndex(oldIndex.getDbName(), oldIndex.getOrigTableName(), oldIndex.getIndexName(), false);
- }
- }
- /**********************************************************************************************
- * StorageDescriptor related methods
- *********************************************************************************************/
-
- /**
- * If this serde has already been read, then return it from the cache. If not, read it, then
- * return it.
- * @param hash hash of the storage descriptor to read
- * @return the storage descriptor
- * @throws IOException
- */
- StorageDescriptor getStorageDescriptor(byte[] hash) throws IOException {
- ByteArrayWrapper hashKey = new ByteArrayWrapper(hash);
- StorageDescriptor cached = sdCache.get(hashKey);
- if (cached != null) return cached;
- LOG.debug("Not found in cache, looking in hbase");
- byte[] serialized = read(SD_TABLE, hash, CATALOG_CF, CATALOG_COL);
- if (serialized == null) {
- throw new RuntimeException("Woh, bad! Trying to fetch a non-existent storage descriptor " +
- "from hash " + Base64.encodeBase64String(hash));
- }
- StorageDescriptor sd = HBaseUtils.deserializeStorageDescriptor(serialized);
- sdCache.put(hashKey, sd);
- return sd;
- }
-
- /**
- * Lower the reference count on the storage descriptor by one. If it goes to zero, then it
- * will be deleted.
- * @param sd Storage descriptor
- * @throws IOException
- */
- void decrementStorageDescriptorRefCount(StorageDescriptor sd) throws IOException {
- byte[] key = HBaseUtils.hashStorageDescriptor(sd, md);
- byte[] serializedRefCnt = read(SD_TABLE, key, CATALOG_CF, REF_COUNT_COL);
- if (serializedRefCnt == null) {
- // Someone deleted it before we got to it, no worries
- return;
- }
- int refCnt = Integer.parseInt(new String(serializedRefCnt, HBaseUtils.ENCODING));
- HTableInterface htab = conn.getHBaseTable(SD_TABLE);
- if (--refCnt < 1) {
- Delete d = new Delete(key);
- // We don't use checkAndDelete here because it isn't compatible with the transaction
- // managers. If the transaction managers are doing their jobs then we should not need it
- // anyway.
- htab.delete(d);
- sdCache.remove(new ByteArrayWrapper(key));
- } else {
- Put p = new Put(key);
- p.add(CATALOG_CF, REF_COUNT_COL, Integer.toString(refCnt).getBytes(HBaseUtils.ENCODING));
- htab.put(p);
- conn.flush(htab);
- }
- }
-
- /**
- * Place the common parts of a storage descriptor into the cache and write the storage
- * descriptor out to HBase. This should only be called if you are sure that the storage
- * descriptor needs to be added. If you have changed a table or partition but not it's storage
- * descriptor do not call this method, as it will increment the reference count of the storage
- * descriptor.
- * @param storageDescriptor storage descriptor to store.
- * @return id of the entry in the cache, to be written in for the storage descriptor
- */
- byte[] putStorageDescriptor(StorageDescriptor storageDescriptor) throws IOException {
- byte[] sd = HBaseUtils.serializeStorageDescriptor(storageDescriptor);
- byte[] key = HBaseUtils.hashStorageDescriptor(storageDescriptor, md);
- byte[] serializedRefCnt = read(SD_TABLE, key, CATALOG_CF, REF_COUNT_COL);
- HTableInterface htab = conn.getHBaseTable(SD_TABLE);
- if (serializedRefCnt == null) {
- // We are the first to put it in the DB
- Put p = new Put(key);
- p.add(CATALOG_CF, CATALOG_COL, sd);
- p.add(CATALOG_CF, REF_COUNT_COL, "1".getBytes(HBaseUtils.ENCODING));
- htab.put(p);
- sdCache.put(new ByteArrayWrapper(key), storageDescriptor);
- } else {
- // Just increment the reference count
- int refCnt = Integer.parseInt(new String(serializedRefCnt, HBaseUtils.ENCODING)) + 1;
- Put p = new Put(key);
- p.add(CATALOG_CF, REF_COUNT_COL, Integer.toString(refCnt).getBytes(HBaseUtils.ENCODING));
- htab.put(p);
- }
- conn.flush(htab);
- return key;
- }
-
- /**
- * Print out a storage descriptor.
- * @param hash hash that is the key of the storage descriptor
- * @return string version of the storage descriptor
- */
- String printStorageDescriptor(byte[] hash) throws IOException, TException {
- byte[] serialized = read(SD_TABLE, hash, CATALOG_CF, CATALOG_COL);
- if (serialized == null) return noSuch(Base64.encodeBase64URLSafeString(hash), "storage descriptor");
- return dumpThriftObject(HBaseUtils.deserializeStorageDescriptor(serialized));
- }
-
- /**
- * Print all of the storage descriptors. This doesn't take a regular expression since the key
- * is an md5 hash and it's hard to see how a regex on this would be useful.
- * @return list of all storage descriptors as strings
- * @throws IOException
- * @throws TException
- */
- List<String> printStorageDescriptors() throws IOException, TException {
- Iterator<Result> results = scan(SD_TABLE, CATALOG_CF, CATALOG_COL);
- if (!results.hasNext()) return Arrays.asList("No storage descriptors");
- List<String> lines = new ArrayList<>();
- while (results.hasNext()) {
- Result result = results.next();
- lines.add(Base64.encodeBase64URLSafeString(result.getRow()) + ": " +
- dumpThriftObject(HBaseUtils.deserializeStorageDescriptor(result.getValue(CATALOG_CF,
- CATALOG_COL))));
- }
- return lines;
- }
-
- private static class ByteArrayWrapper {
- byte[] wrapped;
-
- ByteArrayWrapper(byte[] b) {
- wrapped = b;
- }
-
- @Override
- public boolean equals(Object other) {
- if (other instanceof ByteArrayWrapper) {
- return Arrays.equals(((ByteArrayWrapper)other).wrapped, wrapped);
- } else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(wrapped);
- }
- }
-
- /**********************************************************************************************
- * Statistics related methods
- *********************************************************************************************/
-
- /**
- * Update statistics for one or more columns for a table or a partition.
- *
- * @param dbName database the table is in
- * @param tableName table to update statistics for
- * @param partVals partition values that define partition to update statistics for. If this is
- * null, then these will be assumed to be table level statistics
- * @param stats Stats object with stats for one or more columns
- * @throws IOException
- */
- void updateStatistics(String dbName, String tableName, List<String> partVals,
- ColumnStatistics stats) throws IOException {
- byte[] key = getStatisticsKey(dbName, tableName, partVals);
- String hbaseTable = getStatisticsTable(partVals);
- byte[][] colnames = new byte[stats.getStatsObjSize()][];
- byte[][] serialized = new byte[stats.getStatsObjSize()][];
- for (int i = 0; i < stats.getStatsObjSize(); i++) {
- ColumnStatisticsObj obj = stats.getStatsObj().get(i);
- serialized[i] = HBaseUtils.serializeStatsForOneColumn(stats, obj);
- String colname = obj.getColName();
- colnames[i] = HBaseUtils.buildKey(colname);
- }
- store(hbaseTable, key, STATS_CF, colnames, serialized);
- }
-
- /**
- * Get statistics for a table
- *
- * @param dbName name of database table is in
- * @param tblName name of table
- * @param colNames list of column names to get statistics for
- * @return column statistics for indicated table
- * @throws IOException
- */
- ColumnStatistics getTableStatistics(String dbName, String tblName, List<String> colNames)
- throws IOException {
- byte[] tabKey = HBaseUtils.buildKey(dbName, tblName);
- ColumnStatistics tableStats = new ColumnStatistics();
- ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
- statsDesc.setIsTblLevel(true);
- statsDesc.setDbName(dbName);
- statsDesc.setTableName(tblName);
- tableStats.setStatsDesc(statsDesc);
- byte[][] colKeys = new byte[colNames.size()][];
- for (int i = 0; i < colKeys.length; i++) {
- colKeys[i] = HBaseUtils.buildKey(colNames.get(i));
- }
- Result result = read(TABLE_TABLE, tabKey, STATS_CF, colKeys);
- for (int i = 0; i < colKeys.length; i++) {
- byte[] serializedColStats = result.getValue(STATS_CF, colKeys[i]);
- if (serializedColStats == null) {
- // There were no stats for this column, so skip it
- continue;
- }
- ColumnStatisticsObj obj =
- HBaseUtils.deserializeStatsForOneColumn(tableStats, serializedColStats);
- obj.setColName(colNames.get(i));
- tableStats.addToStatsObj(obj);
- }
- return tableStats;
- }
-
- /**
- * Get statistics for a set of partitions
- *
- * @param dbName name of database table is in
- * @param tblName table partitions are in
- * @param partNames names of the partitions, used only to set values inside the return stats
- * objects
- * @param partVals partition values for each partition, needed because this class doesn't know how
- * to translate from partName to partVals
- * @param colNames column names to fetch stats for. These columns will be fetched for all
- * requested partitions
- * @return list of ColumnStats, one for each partition for which we found at least one column's
- * stats.
- * @throws IOException
- */
- List<ColumnStatistics> getPartitionStatistics(String dbName, String tblName,
- List<String> partNames, List<List<String>> partVals, List<String> colNames)
- throws IOExcept
<TRUNCATED>
[07/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
deleted file mode 100644
index 6b7eb9e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
+++ /dev/null
@@ -1,1781 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.api.ResourceType;
-import org.apache.hadoop.hive.metastore.api.ResourceUri;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.ByteStream.Output;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
-import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDeWithEndPrefix;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hive.common.util.BloomFilter;
-import org.apache.hive.common.util.HiveStringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- * Utility functions
- */
-public class HBaseUtils {
-
- final static Charset ENCODING = StandardCharsets.UTF_8;
- final static char KEY_SEPARATOR = '\u0001';
- final static String KEY_SEPARATOR_STR = new String(new char[] {KEY_SEPARATOR});
-
- static final private Logger LOG = LoggerFactory.getLogger(HBaseUtils.class.getName());
-
- /**
- * Build a key for an object in hbase
- * @param components
- * @return
- */
- static byte[] buildKey(String... components) {
- return buildKey(false, components);
- }
-
- static byte[] buildKeyWithTrailingSeparator(String... components) {
- return buildKey(true, components);
- }
-
- private static byte[] buildKey(boolean trailingSeparator, String... components) {
- String protoKey = StringUtils.join(components, KEY_SEPARATOR);
- if (trailingSeparator) protoKey += KEY_SEPARATOR;
- return protoKey.getBytes(ENCODING);
- }
-
- private static HbaseMetastoreProto.Parameters buildParameters(Map<String, String> params) {
- List<HbaseMetastoreProto.ParameterEntry> entries = new ArrayList<>();
- for (Map.Entry<String, String> e : params.entrySet()) {
- entries.add(
- HbaseMetastoreProto.ParameterEntry.newBuilder()
- .setKey(e.getKey())
- .setValue(e.getValue())
- .build());
- }
- return HbaseMetastoreProto.Parameters.newBuilder()
- .addAllParameter(entries)
- .build();
- }
-
- private static Map<String, String> buildParameters(HbaseMetastoreProto.Parameters protoParams) {
- Map<String, String> params = new HashMap<>();
- for (HbaseMetastoreProto.ParameterEntry pe : protoParams.getParameterList()) {
- params.put(pe.getKey(), pe.getValue());
- }
- return params;
- }
-
-
- private static List<HbaseMetastoreProto.PrincipalPrivilegeSetEntry>
- buildPrincipalPrivilegeSetEntry(Map<String, List<PrivilegeGrantInfo>> entries) {
- List<HbaseMetastoreProto.PrincipalPrivilegeSetEntry> results = new ArrayList<>();
- for (Map.Entry<String, List<PrivilegeGrantInfo>> entry : entries.entrySet()) {
- results.add(HbaseMetastoreProto.PrincipalPrivilegeSetEntry.newBuilder()
- .setPrincipalName(entry.getKey())
- .addAllPrivileges(buildPrivilegeGrantInfo(entry.getValue()))
- .build());
- }
- return results;
- }
-
- private static List<HbaseMetastoreProto.PrivilegeGrantInfo> buildPrivilegeGrantInfo(
- List<PrivilegeGrantInfo> privileges) {
- List<HbaseMetastoreProto.PrivilegeGrantInfo> results = new ArrayList<>();
- for (PrivilegeGrantInfo privilege : privileges) {
- HbaseMetastoreProto.PrivilegeGrantInfo.Builder builder =
- HbaseMetastoreProto.PrivilegeGrantInfo.newBuilder();
- if (privilege.getPrivilege() != null) builder.setPrivilege(privilege.getPrivilege());
- builder.setCreateTime(privilege.getCreateTime());
- if (privilege.getGrantor() != null) builder.setGrantor(privilege.getGrantor());
- if (privilege.getGrantorType() != null) {
- builder.setGrantorType(convertPrincipalTypes(privilege.getGrantorType()));
- }
- builder.setGrantOption(privilege.isGrantOption());
- results.add(builder.build());
- }
- return results;
- }
-
- /**
- * Convert Thrift.PrincipalType to HbaseMetastoreProto.principalType
- * @param type
- * @return
- */
- static HbaseMetastoreProto.PrincipalType convertPrincipalTypes(PrincipalType type) {
- switch (type) {
- case USER: return HbaseMetastoreProto.PrincipalType.USER;
- case ROLE: return HbaseMetastoreProto.PrincipalType.ROLE;
- default: throw new RuntimeException("Unknown principal type " + type.toString());
- }
- }
-
- /**
- * Convert principalType from HbaseMetastoreProto to Thrift.PrincipalType
- * @param type
- * @return
- */
- static PrincipalType convertPrincipalTypes(HbaseMetastoreProto.PrincipalType type) {
- switch (type) {
- case USER: return PrincipalType.USER;
- case ROLE: return PrincipalType.ROLE;
- default: throw new RuntimeException("Unknown principal type " + type.toString());
- }
- }
-
- private static Map<String, List<PrivilegeGrantInfo>> convertPrincipalPrivilegeSetEntries(
- List<HbaseMetastoreProto.PrincipalPrivilegeSetEntry> entries) {
- Map<String, List<PrivilegeGrantInfo>> map = new HashMap<>();
- for (HbaseMetastoreProto.PrincipalPrivilegeSetEntry entry : entries) {
- map.put(entry.getPrincipalName(), convertPrivilegeGrantInfos(entry.getPrivilegesList()));
- }
- return map;
- }
-
- private static List<PrivilegeGrantInfo> convertPrivilegeGrantInfos(
- List<HbaseMetastoreProto.PrivilegeGrantInfo> privileges) {
- List<PrivilegeGrantInfo> results = new ArrayList<>();
- for (HbaseMetastoreProto.PrivilegeGrantInfo proto : privileges) {
- PrivilegeGrantInfo pgi = new PrivilegeGrantInfo();
- if (proto.hasPrivilege()) pgi.setPrivilege(proto.getPrivilege());
- pgi.setCreateTime((int)proto.getCreateTime());
- if (proto.hasGrantor()) pgi.setGrantor(proto.getGrantor());
- if (proto.hasGrantorType()) {
- pgi.setGrantorType(convertPrincipalTypes(proto.getGrantorType()));
- }
- if (proto.hasGrantOption()) pgi.setGrantOption(proto.getGrantOption());
- results.add(pgi);
- }
- return results;
- }
-
- private static HbaseMetastoreProto.PrincipalPrivilegeSet
- buildPrincipalPrivilegeSet(PrincipalPrivilegeSet pps) {
- HbaseMetastoreProto.PrincipalPrivilegeSet.Builder builder =
- HbaseMetastoreProto.PrincipalPrivilegeSet.newBuilder();
- if (pps.getUserPrivileges() != null) {
- builder.addAllUsers(buildPrincipalPrivilegeSetEntry(pps.getUserPrivileges()));
- }
- if (pps.getRolePrivileges() != null) {
- builder.addAllRoles(buildPrincipalPrivilegeSetEntry(pps.getRolePrivileges()));
- }
- return builder.build();
- }
-
- private static PrincipalPrivilegeSet buildPrincipalPrivilegeSet(
- HbaseMetastoreProto.PrincipalPrivilegeSet proto) throws InvalidProtocolBufferException {
- PrincipalPrivilegeSet pps = null;
- if (!proto.getUsersList().isEmpty() || !proto.getRolesList().isEmpty()) {
- pps = new PrincipalPrivilegeSet();
- if (!proto.getUsersList().isEmpty()) {
- pps.setUserPrivileges(convertPrincipalPrivilegeSetEntries(proto.getUsersList()));
- }
- if (!proto.getRolesList().isEmpty()) {
- pps.setRolePrivileges(convertPrincipalPrivilegeSetEntries(proto.getRolesList()));
- }
- }
- return pps;
- }
- /**
- * Serialize a PrincipalPrivilegeSet
- * @param pps
- * @return
- */
- static byte[] serializePrincipalPrivilegeSet(PrincipalPrivilegeSet pps) {
- return buildPrincipalPrivilegeSet(pps).toByteArray();
- }
-
- /**
- * Deserialize a PrincipalPrivilegeSet
- * @param serialized
- * @return
- * @throws InvalidProtocolBufferException
- */
- static PrincipalPrivilegeSet deserializePrincipalPrivilegeSet(byte[] serialized)
- throws InvalidProtocolBufferException {
- HbaseMetastoreProto.PrincipalPrivilegeSet proto =
- HbaseMetastoreProto.PrincipalPrivilegeSet.parseFrom(serialized);
- return buildPrincipalPrivilegeSet(proto);
- }
-
- /**
- * Serialize a role
- * @param role
- * @return two byte arrays, first contains the key, the second the serialized value.
- */
- static byte[][] serializeRole(Role role) {
- byte[][] result = new byte[2][];
- result[0] = buildKey(role.getRoleName());
- HbaseMetastoreProto.Role.Builder builder = HbaseMetastoreProto.Role.newBuilder();
- builder.setCreateTime(role.getCreateTime());
- if (role.getOwnerName() != null) builder.setOwnerName(role.getOwnerName());
- result[1] = builder.build().toByteArray();
- return result;
- }
-
- /**
- * Deserialize a role. This method should be used when the rolename is already known as it
- * doesn't have to re-deserialize it.
- * @param roleName name of the role
- * @param value value fetched from hbase
- * @return A role
- * @throws InvalidProtocolBufferException
- */
- static Role deserializeRole(String roleName, byte[] value)
- throws InvalidProtocolBufferException {
- Role role = new Role();
- role.setRoleName(roleName);
- HbaseMetastoreProto.Role protoRole =
- HbaseMetastoreProto.Role.parseFrom(value);
- role.setCreateTime((int)protoRole.getCreateTime());
- if (protoRole.hasOwnerName()) role.setOwnerName(protoRole.getOwnerName());
- return role;
- }
-
- /**
- * Deserialize a role. This method should be used when the rolename is not already known (eg
- * when doing a scan).
- * @param key key from hbase
- * @param value value from hbase
- * @return a role
- * @throws InvalidProtocolBufferException
- */
- static Role deserializeRole(byte[] key, byte[] value)
- throws InvalidProtocolBufferException {
- String roleName = new String(key, ENCODING);
- return deserializeRole(roleName, value);
- }
-
- /**
- * Serialize a list of role names
- * @param roles
- * @return
- */
- static byte[] serializeRoleList(List<String> roles) {
- return HbaseMetastoreProto.RoleList.newBuilder()
- .addAllRole(roles)
- .build()
- .toByteArray();
- }
-
- static List<String> deserializeRoleList(byte[] value) throws InvalidProtocolBufferException {
- HbaseMetastoreProto.RoleList proto = HbaseMetastoreProto.RoleList.parseFrom(value);
- return new ArrayList<>(proto.getRoleList());
- }
-
- /**
- * Serialize a database
- * @param db
- * @return two byte arrays, first contains the key, the second the serialized value.
- */
- static byte[][] serializeDatabase(Database db) {
- byte[][] result = new byte[2][];
- result[0] = buildKey(HiveStringUtils.normalizeIdentifier(db.getName()));
- HbaseMetastoreProto.Database.Builder builder = HbaseMetastoreProto.Database.newBuilder();
-
- if (db.getDescription() != null) builder.setDescription(db.getDescription());
- if (db.getLocationUri() != null) builder.setUri(db.getLocationUri());
- if (db.getParameters() != null) builder.setParameters(buildParameters(db.getParameters()));
- if (db.getPrivileges() != null) {
- builder.setPrivileges(buildPrincipalPrivilegeSet(db.getPrivileges()));
- }
- if (db.getOwnerName() != null) builder.setOwnerName(db.getOwnerName());
- if (db.getOwnerType() != null) builder.setOwnerType(convertPrincipalTypes(db.getOwnerType()));
-
- result[1] = builder.build().toByteArray();
- return result;
- }
-
-
- /**
- * Deserialize a database. This method should be used when the db anme is already known as it
- * doesn't have to re-deserialize it.
- * @param dbName name of the role
- * @param value value fetched from hbase
- * @return A database
- * @throws InvalidProtocolBufferException
- */
- static Database deserializeDatabase(String dbName, byte[] value)
- throws InvalidProtocolBufferException {
- Database db = new Database();
- db.setName(dbName);
- HbaseMetastoreProto.Database protoDb = HbaseMetastoreProto.Database.parseFrom(value);
- if (protoDb.hasDescription()) db.setDescription(protoDb.getDescription());
- if (protoDb.hasUri()) db.setLocationUri(protoDb.getUri());
- if (protoDb.hasParameters()) db.setParameters(buildParameters(protoDb.getParameters()));
- if (protoDb.hasPrivileges()) {
- db.setPrivileges(buildPrincipalPrivilegeSet(protoDb.getPrivileges()));
- }
- if (protoDb.hasOwnerName()) db.setOwnerName(protoDb.getOwnerName());
- if (protoDb.hasOwnerType()) db.setOwnerType(convertPrincipalTypes(protoDb.getOwnerType()));
-
- return db;
- }
-
- /**
- * Deserialize a database. This method should be used when the db name is not already known (eg
- * when doing a scan).
- * @param key key from hbase
- * @param value value from hbase
- * @return a role
- * @throws InvalidProtocolBufferException
- */
- static Database deserializeDatabase(byte[] key, byte[] value)
- throws InvalidProtocolBufferException {
- String dbName = new String(key, ENCODING);
- return deserializeDatabase(dbName, value);
- }
-
- /**
- * Serialize a function
- * @param func function to serialize
- * @return two byte arrays, first contains the key, the second the value.
- */
- static byte[][] serializeFunction(Function func) {
- byte[][] result = new byte[2][];
- result[0] = buildKey(func.getDbName(), func.getFunctionName());
- HbaseMetastoreProto.Function.Builder builder = HbaseMetastoreProto.Function.newBuilder();
- if (func.getClassName() != null) builder.setClassName(func.getClassName());
- if (func.getOwnerName() != null) builder.setOwnerName(func.getOwnerName());
- if (func.getOwnerType() != null) {
- builder.setOwnerType(convertPrincipalTypes(func.getOwnerType()));
- }
- builder.setCreateTime(func.getCreateTime());
- if (func.getFunctionType() != null) {
- builder.setFunctionType(convertFunctionTypes(func.getFunctionType()));
- }
- if (func.getResourceUris() != null) {
- for (ResourceUri uri : func.getResourceUris()) {
- builder.addResourceUris(HbaseMetastoreProto.Function.ResourceUri.newBuilder()
- .setResourceType(convertResourceTypes(uri.getResourceType()))
- .setUri(uri.getUri()));
- }
- }
- result[1] = builder.build().toByteArray();
- return result;
- }
-
- /**
- * Deserialize a function. This method should be used when the function and db name are
- * already known.
- * @param dbName name of the database the function is in
- * @param functionName name of the function
- * @param value serialized value of the function
- * @return function as an object
- * @throws InvalidProtocolBufferException
- */
- static Function deserializeFunction(String dbName, String functionName, byte[] value)
- throws InvalidProtocolBufferException {
- Function func = new Function();
- func.setDbName(dbName);
- func.setFunctionName(functionName);
- HbaseMetastoreProto.Function protoFunc = HbaseMetastoreProto.Function.parseFrom(value);
- if (protoFunc.hasClassName()) func.setClassName(protoFunc.getClassName());
- if (protoFunc.hasOwnerName()) func.setOwnerName(protoFunc.getOwnerName());
- if (protoFunc.hasOwnerType()) {
- func.setOwnerType(convertPrincipalTypes(protoFunc.getOwnerType()));
- }
- func.setCreateTime((int)protoFunc.getCreateTime());
- if (protoFunc.hasFunctionType()) {
- func.setFunctionType(convertFunctionTypes(protoFunc.getFunctionType()));
- }
- for (HbaseMetastoreProto.Function.ResourceUri protoUri : protoFunc.getResourceUrisList()) {
- func.addToResourceUris(new ResourceUri(convertResourceTypes(protoUri.getResourceType()),
- protoUri.getUri()));
- }
- return func;
- }
-
- /**
- * Deserialize a function. This method should be used when the dbname and function name are
- * not already known, such as in a scan.
- * @param key key from hbase
- * @param value value from hbase
- * @return function object
- * @throws InvalidProtocolBufferException
- */
- static Function deserializeFunction(byte[] key, byte[] value)
- throws InvalidProtocolBufferException {
- String[] keys = deserializeKey(key);
- return deserializeFunction(keys[0], keys[1], value);
- }
-
- private static HbaseMetastoreProto.Function.FunctionType convertFunctionTypes(FunctionType type) {
- switch (type) {
- case JAVA: return HbaseMetastoreProto.Function.FunctionType.JAVA;
- default: throw new RuntimeException("Unknown function type " + type.toString());
- }
- }
-
- private static FunctionType convertFunctionTypes(HbaseMetastoreProto.Function.FunctionType type) {
- switch (type) {
- case JAVA: return FunctionType.JAVA;
- default: throw new RuntimeException("Unknown function type " + type.toString());
- }
- }
-
- private static HbaseMetastoreProto.Function.ResourceUri.ResourceType
- convertResourceTypes(ResourceType type) {
- switch (type) {
- case JAR: return HbaseMetastoreProto.Function.ResourceUri.ResourceType.JAR;
- case FILE: return HbaseMetastoreProto.Function.ResourceUri.ResourceType.FILE;
- case ARCHIVE: return HbaseMetastoreProto.Function.ResourceUri.ResourceType.ARCHIVE;
- default: throw new RuntimeException("Unknown resource type " + type.toString());
- }
- }
-
- private static ResourceType convertResourceTypes(
- HbaseMetastoreProto.Function.ResourceUri.ResourceType type) {
- switch (type) {
- case JAR: return ResourceType.JAR;
- case FILE: return ResourceType.FILE;
- case ARCHIVE: return ResourceType.ARCHIVE;
- default: throw new RuntimeException("Unknown resource type " + type.toString());
- }
- }
-
- private static List<FieldSchema>
- convertFieldSchemaListFromProto(List<HbaseMetastoreProto.FieldSchema> protoList) {
- List<FieldSchema> schemas = new ArrayList<>(protoList.size());
- for (HbaseMetastoreProto.FieldSchema proto : protoList) {
- schemas.add(new FieldSchema(proto.getName(), proto.getType(),
- proto.hasComment() ? proto.getComment() : null));
- }
- return schemas;
- }
-
- private static List<HbaseMetastoreProto.FieldSchema>
- convertFieldSchemaListToProto(List<FieldSchema> schemas) {
- List<HbaseMetastoreProto.FieldSchema> protoList = new ArrayList<>(schemas.size());
- for (FieldSchema fs : schemas) {
- HbaseMetastoreProto.FieldSchema.Builder builder =
- HbaseMetastoreProto.FieldSchema.newBuilder();
- builder
- .setName(fs.getName())
- .setType(fs.getType());
- if (fs.getComment() != null) builder.setComment(fs.getComment());
- protoList.add(builder.build());
- }
- return protoList;
- }
-
- /**
- * Serialize a storage descriptor.
- * @param sd storage descriptor to serialize
- * @return serialized storage descriptor.
- */
- static byte[] serializeStorageDescriptor(StorageDescriptor sd) {
- HbaseMetastoreProto.StorageDescriptor.Builder builder =
- HbaseMetastoreProto.StorageDescriptor.newBuilder();
- builder.addAllCols(convertFieldSchemaListToProto(sd.getCols()));
- if (sd.getInputFormat() != null) {
- builder.setInputFormat(sd.getInputFormat());
- }
- if (sd.getOutputFormat() != null) {
- builder.setOutputFormat(sd.getOutputFormat());
- }
- builder.setIsCompressed(sd.isCompressed());
- builder.setNumBuckets(sd.getNumBuckets());
- if (sd.getSerdeInfo() != null) {
- HbaseMetastoreProto.StorageDescriptor.SerDeInfo.Builder serdeBuilder =
- HbaseMetastoreProto.StorageDescriptor.SerDeInfo.newBuilder();
- SerDeInfo serde = sd.getSerdeInfo();
- if (serde.getName() != null) {
- serdeBuilder.setName(serde.getName());
- }
- if (serde.getSerializationLib() != null) {
- serdeBuilder.setSerializationLib(serde.getSerializationLib());
- }
- if (serde.getParameters() != null) {
- serdeBuilder.setParameters(buildParameters(serde.getParameters()));
- }
- builder.setSerdeInfo(serdeBuilder);
- }
- if (sd.getBucketCols() != null) {
- builder.addAllBucketCols(sd.getBucketCols());
- }
- if (sd.getSortCols() != null) {
- List<Order> orders = sd.getSortCols();
- List<HbaseMetastoreProto.StorageDescriptor.Order> protoList = new ArrayList<>(orders.size());
- for (Order order : orders) {
- protoList.add(HbaseMetastoreProto.StorageDescriptor.Order.newBuilder()
- .setColumnName(order.getCol())
- .setOrder(order.getOrder())
- .build());
- }
- builder.addAllSortCols(protoList);
- }
- if (sd.getSkewedInfo() != null) {
- HbaseMetastoreProto.StorageDescriptor.SkewedInfo.Builder skewBuilder =
- HbaseMetastoreProto.StorageDescriptor.SkewedInfo.newBuilder();
- SkewedInfo skewed = sd.getSkewedInfo();
- if (skewed.getSkewedColNames() != null) {
- skewBuilder.addAllSkewedColNames(skewed.getSkewedColNames());
- }
- if (skewed.getSkewedColValues() != null) {
- for (List<String> innerList : skewed.getSkewedColValues()) {
- HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueList.Builder listBuilder =
- HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueList.newBuilder();
- listBuilder.addAllSkewedColValue(innerList);
- skewBuilder.addSkewedColValues(listBuilder);
- }
- }
- if (skewed.getSkewedColValueLocationMaps() != null) {
- for (Map.Entry<List<String>, String> e : skewed.getSkewedColValueLocationMaps().entrySet()) {
- HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueLocationMap.Builder mapBuilder =
- HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueLocationMap.newBuilder();
- mapBuilder.addAllKey(e.getKey());
- mapBuilder.setValue(e.getValue());
- skewBuilder.addSkewedColValueLocationMaps(mapBuilder);
- }
- }
- builder.setSkewedInfo(skewBuilder);
- }
- builder.setStoredAsSubDirectories(sd.isStoredAsSubDirectories());
-
- return builder.build().toByteArray();
- }
-
- /**
- * Produce a hash for the storage descriptor
- * @param sd storage descriptor to hash
- * @param md message descriptor to use to generate the hash
- * @return the hash as a byte array
- */
- public static byte[] hashStorageDescriptor(StorageDescriptor sd, MessageDigest md) {
- // Note all maps and lists have to be absolutely sorted. Otherwise we'll produce different
- // results for hashes based on the OS or JVM being used.
- md.reset();
- for (FieldSchema fs : sd.getCols()) {
- md.update(fs.getName().getBytes(ENCODING));
- md.update(fs.getType().getBytes(ENCODING));
- if (fs.getComment() != null) md.update(fs.getComment().getBytes(ENCODING));
- }
- if (sd.getInputFormat() != null) {
- md.update(sd.getInputFormat().getBytes(ENCODING));
- }
- if (sd.getOutputFormat() != null) {
- md.update(sd.getOutputFormat().getBytes(ENCODING));
- }
- md.update(sd.isCompressed() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
- md.update(Integer.toString(sd.getNumBuckets()).getBytes(ENCODING));
- if (sd.getSerdeInfo() != null) {
- SerDeInfo serde = sd.getSerdeInfo();
- if (serde.getName() != null) {
- md.update(serde.getName().getBytes(ENCODING));
- }
- if (serde.getSerializationLib() != null) {
- md.update(serde.getSerializationLib().getBytes(ENCODING));
- }
- if (serde.getParameters() != null) {
- SortedMap<String, String> params = new TreeMap<>(serde.getParameters());
- for (Map.Entry<String, String> param : params.entrySet()) {
- md.update(param.getKey().getBytes(ENCODING));
- md.update(param.getValue().getBytes(ENCODING));
- }
- }
- }
- if (sd.getBucketCols() != null) {
- List<String> bucketCols = new ArrayList<>(sd.getBucketCols());
- for (String bucket : bucketCols) md.update(bucket.getBytes(ENCODING));
- }
- if (sd.getSortCols() != null) {
- SortedSet<Order> orders = new TreeSet<>(sd.getSortCols());
- for (Order order : orders) {
- md.update(order.getCol().getBytes(ENCODING));
- md.update(Integer.toString(order.getOrder()).getBytes(ENCODING));
- }
- }
- if (sd.getSkewedInfo() != null) {
- SkewedInfo skewed = sd.getSkewedInfo();
- if (skewed.getSkewedColNames() != null) {
- SortedSet<String> colnames = new TreeSet<>(skewed.getSkewedColNames());
- for (String colname : colnames) md.update(colname.getBytes(ENCODING));
- }
- if (skewed.getSkewedColValues() != null) {
- SortedSet<String> sortedOuterList = new TreeSet<>();
- for (List<String> innerList : skewed.getSkewedColValues()) {
- SortedSet<String> sortedInnerList = new TreeSet<>(innerList);
- sortedOuterList.add(StringUtils.join(sortedInnerList, "."));
- }
- for (String colval : sortedOuterList) md.update(colval.getBytes(ENCODING));
- }
- if (skewed.getSkewedColValueLocationMaps() != null) {
- SortedMap<String, String> sortedMap = new TreeMap<>();
- for (Map.Entry<List<String>, String> smap : skewed.getSkewedColValueLocationMaps().entrySet()) {
- SortedSet<String> sortedKey = new TreeSet<>(smap.getKey());
- sortedMap.put(StringUtils.join(sortedKey, "."), smap.getValue());
- }
- for (Map.Entry<String, String> e : sortedMap.entrySet()) {
- md.update(e.getKey().getBytes(ENCODING));
- md.update(e.getValue().getBytes(ENCODING));
- }
- }
- md.update(sd.isStoredAsSubDirectories() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
- }
-
- return md.digest();
- }
-
- static StorageDescriptor deserializeStorageDescriptor(byte[] serialized)
- throws InvalidProtocolBufferException {
- HbaseMetastoreProto.StorageDescriptor proto =
- HbaseMetastoreProto.StorageDescriptor.parseFrom(serialized);
- StorageDescriptor sd = new StorageDescriptor();
- sd.setCols(convertFieldSchemaListFromProto(proto.getColsList()));
- if (proto.hasInputFormat()) sd.setInputFormat(proto.getInputFormat());
- if (proto.hasOutputFormat()) sd.setOutputFormat(proto.getOutputFormat());
- sd.setCompressed(proto.getIsCompressed());
- sd.setNumBuckets(proto.getNumBuckets());
- if (proto.hasSerdeInfo()) {
- SerDeInfo serde = new SerDeInfo();
- serde.setName(proto.getSerdeInfo().hasName()?
- proto.getSerdeInfo().getName():null);
- serde.setSerializationLib(proto.getSerdeInfo().hasSerializationLib()?
- proto.getSerdeInfo().getSerializationLib():null);
- serde.setParameters(buildParameters(proto.getSerdeInfo().getParameters()));
- sd.setSerdeInfo(serde);
- }
- sd.setBucketCols(new ArrayList<>(proto.getBucketColsList()));
- List<Order> sortCols = new ArrayList<>();
- for (HbaseMetastoreProto.StorageDescriptor.Order protoOrder : proto.getSortColsList()) {
- sortCols.add(new Order(protoOrder.getColumnName(), protoOrder.getOrder()));
- }
- sd.setSortCols(sortCols);
- if (proto.hasSkewedInfo()) {
- SkewedInfo skewed = new SkewedInfo();
- skewed
- .setSkewedColNames(new ArrayList<>(proto.getSkewedInfo().getSkewedColNamesList()));
- for (HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueList innerList :
- proto.getSkewedInfo().getSkewedColValuesList()) {
- skewed.addToSkewedColValues(new ArrayList<>(innerList.getSkewedColValueList()));
- }
- Map<List<String>, String> colMaps = new HashMap<>();
- for (HbaseMetastoreProto.StorageDescriptor.SkewedInfo.SkewedColValueLocationMap map :
- proto.getSkewedInfo().getSkewedColValueLocationMapsList()) {
- colMaps.put(new ArrayList<>(map.getKeyList()), map.getValue());
- }
- skewed.setSkewedColValueLocationMaps(colMaps);
- sd.setSkewedInfo(skewed);
- }
- if (proto.hasStoredAsSubDirectories()) {
- sd.setStoredAsSubDirectories(proto.getStoredAsSubDirectories());
- }
- return sd;
- }
-
- static List<String> getPartitionKeyTypes(List<FieldSchema> parts) {
- com.google.common.base.Function<FieldSchema, String> fieldSchemaToType =
- new com.google.common.base.Function<FieldSchema, String>() {
- public String apply(FieldSchema fs) { return fs.getType(); }
- };
- return Lists.transform(parts, fieldSchemaToType);
- }
-
- static List<String> getPartitionNames(List<FieldSchema> parts) {
- com.google.common.base.Function<FieldSchema, String> fieldSchemaToName =
- new com.google.common.base.Function<FieldSchema, String>() {
- public String apply(FieldSchema fs) { return fs.getName(); }
- };
- return Lists.transform(parts, fieldSchemaToName);
- }
-
- /**
- * Serialize a partition
- * @param part partition object
- * @param sdHash hash that is being used as a key for the enclosed storage descriptor
- * @return First element is the key, second is the serialized partition
- */
- static byte[][] serializePartition(Partition part, List<String> partTypes, byte[] sdHash) {
- byte[][] result = new byte[2][];
- result[0] = buildPartitionKey(part.getDbName(), part.getTableName(), partTypes, part.getValues());
- HbaseMetastoreProto.Partition.Builder builder = HbaseMetastoreProto.Partition.newBuilder();
- builder
- .setCreateTime(part.getCreateTime())
- .setLastAccessTime(part.getLastAccessTime());
- if (part.getSd().getLocation() != null) builder.setLocation(part.getSd().getLocation());
- if (part.getSd().getParameters() != null) {
- builder.setSdParameters(buildParameters(part.getSd().getParameters()));
- }
- builder.setSdHash(ByteString.copyFrom(sdHash));
- if (part.getParameters() != null) builder.setParameters(buildParameters(part.getParameters()));
- result[1] = builder.build().toByteArray();
- return result;
- }
-
- static byte[] buildPartitionKey(String dbName, String tableName, List<String> partTypes, List<String> partVals) {
- return buildPartitionKey(dbName, tableName, partTypes, partVals, false);
- }
-
- static byte[] buildPartitionKey(String dbName, String tableName, List<String> partTypes, List<String> partVals, boolean endPrefix) {
- Object[] components = new Object[partVals.size()];
- for (int i=0;i<partVals.size();i++) {
- TypeInfo expectedType =
- TypeInfoUtils.getTypeInfoFromTypeString(partTypes.get(i));
- ObjectInspector outputOI =
- TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(expectedType);
- Converter converter = ObjectInspectorConverters.getConverter(
- PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
- components[i] = converter.convert(partVals.get(i));
- }
-
- return buildSerializedPartitionKey(dbName, tableName, partTypes, components, endPrefix);
- }
-
- static byte[] buildSerializedPartitionKey(String dbName, String tableName, List<String> partTypes, Object[] components, boolean endPrefix) {
- ObjectInspector javaStringOI =
- PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveCategory.STRING);
- Object[] data = new Object[components.length+2];
- List<ObjectInspector> fois = new ArrayList<ObjectInspector>(components.length+2);
- boolean[] endPrefixes = new boolean[components.length+2];
-
- data[0] = dbName;
- fois.add(javaStringOI);
- endPrefixes[0] = false;
- data[1] = tableName;
- fois.add(javaStringOI);
- endPrefixes[1] = false;
-
- for (int i = 0; i < components.length; i++) {
- data[i+2] = components[i];
- TypeInfo expectedType =
- TypeInfoUtils.getTypeInfoFromTypeString(partTypes.get(i));
- ObjectInspector outputOI =
- TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(expectedType);
- fois.add(outputOI);
- }
- Output output = new Output();
- try {
- BinarySortableSerDeWithEndPrefix.serializeStruct(output, data, fois, endPrefix);
- } catch (SerDeException e) {
- throw new RuntimeException("Cannot serialize partition " + StringUtils.join(components, ","));
- }
- return Arrays.copyOf(output.getData(), output.getLength());
- }
-
- static class StorageDescriptorParts {
- byte[] sdHash;
- String location;
- Map<String, String> parameters;
- Partition containingPartition;
- Table containingTable;
- Index containingIndex;
- }
-
- static void assembleStorageDescriptor(StorageDescriptor sd, StorageDescriptorParts parts) {
- SharedStorageDescriptor ssd = new SharedStorageDescriptor();
- ssd.setLocation(parts.location);
- ssd.setParameters(parts.parameters);
- ssd.setShared(sd);
- if (parts.containingPartition != null) {
- parts.containingPartition.setSd(ssd);
- } else if (parts.containingTable != null) {
- parts.containingTable.setSd(ssd);
- } else if (parts.containingIndex != null) {
- parts.containingIndex.setSd(ssd);
- }
- else {
- throw new RuntimeException("Need either a partition or a table");
- }
- }
-
- /**
- * Deserialize a partition key when you know nothing about it. That is, you do not know what
- * dbname, tablename it came from.
- * @param key the key fetched from HBase
- * @param callback A reference to the calling HBaseReadWrite object. This has to be done as a
- * callback because we have to first deserialize the database name and table
- * name, and then fetch the table information, and then we will know how to
- * desierliaze the rest of the key.
- * @return a list that includes the dbname, tablename, and partition values
- * @throws IOException
- */
- static List<String> deserializePartitionKey(byte[] key, HBaseReadWrite callback)
- throws IOException {
- List<String> keyParts =
- desierliazeDbNameTableNameFromPartitionKey(key, callback.getConf());
- Table table = callback.getTable(keyParts.get(0), keyParts.get(1));
- keyParts.addAll(deserializePartitionKey(table.getPartitionKeys(), key, callback.getConf()));
- return keyParts;
- }
-
- /**
- * Deserialize a partition. This version should be used when the partition key is not already
- * known and the database and table name are not known either (eg a full scan). Because the
- * dbname and tablename (and thus the partition columns) are not known a priori this version
- * has to go fetch the table after it figures out which table. If you already have the table
- * object you should use
- * {@link #deserializePartition(String,String,List,byte[],byte[],Configuration)}
- * @param key the key fetched from HBase
- * @param serialized the value fetched from HBase
- * @param callback A reference to the calling HBaseReadWrite object. This has to be done as a
- * callback because we have to first deserialize the database name and table
- * name, and then fetch the table information, and then we will know how to
- * desierliaze the rest of the key.
- * @return A struct that contains the partition plus parts of the storage descriptor
- */
- static StorageDescriptorParts deserializePartition(byte[] key, byte[] serialized,
- HBaseReadWrite callback)
- throws IOException {
- List<String> dbNameTableName =
- desierliazeDbNameTableNameFromPartitionKey(key, callback.getConf());
- Table table = callback.getTable(dbNameTableName.get(0), dbNameTableName.get(1));
- List<String> keys = deserializePartitionKey(table.getPartitionKeys(), key, callback.getConf());
- return deserializePartition(dbNameTableName.get(0), dbNameTableName.get(1), keys, serialized);
- }
-
- /**
- * Deserialize a partition. This version should be used when you know the dbname and tablename
- * but not the partition values.
- * @param dbName database this partition is in
- * @param tableName table this partition is in
- * @param partitions schemas for the partition columns of this table
- * @param key key fetched from HBase
- * @param serialized serialized version of the partition
- * @param conf configuration file
- * @return
- * @throws InvalidProtocolBufferException
- */
- static StorageDescriptorParts deserializePartition(String dbName, String tableName,
- List<FieldSchema> partitions, byte[] key,
- byte[] serialized, Configuration conf)
- throws InvalidProtocolBufferException {
- List<String> keys = deserializePartitionKey(partitions, key, conf);
- return deserializePartition(dbName, tableName, keys, serialized);
- }
-
- /**
- * Deserialize a partition. This version should be used when the partition key is
- * known (eg a get).
- * @param dbName database name
- * @param tableName table name
- * @param partVals partition values
- * @param serialized the value fetched from HBase
- * @return A struct that contains the partition plus parts of the storage descriptor
- */
- static StorageDescriptorParts deserializePartition(String dbName, String tableName,
- List<String> partVals, byte[] serialized)
- throws InvalidProtocolBufferException {
- HbaseMetastoreProto.Partition proto = HbaseMetastoreProto.Partition.parseFrom(serialized);
- Partition part = new Partition();
- StorageDescriptorParts sdParts = new StorageDescriptorParts();
- sdParts.containingPartition = part;
- part.setDbName(dbName);
- part.setTableName(tableName);
- part.setValues(partVals);
- part.setCreateTime((int)proto.getCreateTime());
- part.setLastAccessTime((int)proto.getLastAccessTime());
- if (proto.hasLocation()) sdParts.location = proto.getLocation();
- if (proto.hasSdParameters()) sdParts.parameters = buildParameters(proto.getSdParameters());
- sdParts.sdHash = proto.getSdHash().toByteArray();
- if (proto.hasParameters()) part.setParameters(buildParameters(proto.getParameters()));
- return sdParts;
- }
-
- static String[] deserializeKey(byte[] key) {
- String k = new String(key, ENCODING);
- return k.split(KEY_SEPARATOR_STR);
- }
-
- private static List<String> desierliazeDbNameTableNameFromPartitionKey(byte[] key,
- Configuration conf) {
- StringBuilder names = new StringBuilder();
- names.append("dbName,tableName,");
- StringBuilder types = new StringBuilder();
- types.append("string,string,");
- BinarySortableSerDe serDe = new BinarySortableSerDe();
- Properties props = new Properties();
- props.setProperty(serdeConstants.LIST_COLUMNS, names.toString());
- props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types.toString());
- try {
- serDe.initialize(conf, props);
- List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(key))).subList(0, 2);
- List<String> keys = new ArrayList<>();
- for (int i=0;i<deserializedkeys.size();i++) {
- Object deserializedKey = deserializedkeys.get(i);
- if (deserializedKey==null) {
- throw new RuntimeException("Can't have a null dbname or tablename");
- } else {
- TypeInfo inputType = TypeInfoUtils.getTypeInfoFromTypeString("string");
- ObjectInspector inputOI =
- TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(inputType);
- Converter converter = ObjectInspectorConverters.getConverter(inputOI,
- PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- keys.add((String) converter.convert(deserializedKey));
- }
- }
- return keys;
- } catch (SerDeException e) {
- throw new RuntimeException("Error when deserialize key", e);
- }
- }
-
- // Deserialize a partition key and return _only_ the partition values.
- private static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key,
- Configuration conf) {
- StringBuilder names = new StringBuilder();
- names.append("dbName,tableName,");
- StringBuilder types = new StringBuilder();
- types.append("string,string,");
- for (int i=0;i<partitions.size();i++) {
- names.append(partitions.get(i).getName());
- types.append(TypeInfoUtils.getTypeInfoFromTypeString(partitions.get(i).getType()));
- if (i!=partitions.size()-1) {
- names.append(",");
- types.append(",");
- }
- }
- BinarySortableSerDe serDe = new BinarySortableSerDe();
- Properties props = new Properties();
- props.setProperty(serdeConstants.LIST_COLUMNS, names.toString());
- props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types.toString());
- try {
- serDe.initialize(conf, props);
- List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(key))).subList(2, partitions.size()+2);
- List<String> partitionKeys = new ArrayList<String>();
- for (int i=0;i<deserializedkeys.size();i++) {
- Object deserializedKey = deserializedkeys.get(i);
- if (deserializedKey==null) {
- partitionKeys.add(HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULTPARTITIONNAME));
- } else {
- TypeInfo inputType =
- TypeInfoUtils.getTypeInfoFromTypeString(partitions.get(i).getType());
- ObjectInspector inputOI =
- TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(inputType);
- Converter converter = ObjectInspectorConverters.getConverter(inputOI,
- PrimitiveObjectInspectorFactory.javaStringObjectInspector);
- partitionKeys.add((String)converter.convert(deserializedKey));
- }
- }
- return partitionKeys;
- } catch (SerDeException e) {
- throw new RuntimeException("Error when deserialize key", e);
- }
- }
-
- /**
- * Serialize a table
- * @param table table object
- * @param sdHash hash that is being used as a key for the enclosed storage descriptor
- * @return First element is the key, second is the serialized table
- */
- static byte[][] serializeTable(Table table, byte[] sdHash) {
- byte[][] result = new byte[2][];
- result[0] = buildKey(HiveStringUtils.normalizeIdentifier(table.getDbName()),
- HiveStringUtils.normalizeIdentifier(table.getTableName()));
- HbaseMetastoreProto.Table.Builder builder = HbaseMetastoreProto.Table.newBuilder();
- if (table.getOwner() != null) builder.setOwner(table.getOwner());
- builder
- .setCreateTime(table.getCreateTime())
- .setLastAccessTime(table.getLastAccessTime())
- .setRetention(table.getRetention());
- if (table.getSd().getLocation() != null) builder.setLocation(table.getSd().getLocation());
- if (table.getSd().getParameters() != null) {
- builder.setSdParameters(buildParameters(table.getSd().getParameters()));
- }
- builder.setSdHash(ByteString.copyFrom(sdHash));
- if (table.getPartitionKeys() != null) {
- builder.addAllPartitionKeys(convertFieldSchemaListToProto(table.getPartitionKeys()));
- }
- if (table.getParameters() != null) {
- builder.setParameters(buildParameters(table.getParameters()));
- }
- if (table.getViewOriginalText() != null) {
- builder.setViewOriginalText(table.getViewOriginalText());
- }
- if (table.getViewExpandedText() != null) {
- builder.setViewExpandedText(table.getViewExpandedText());
- }
- builder.setIsRewriteEnabled(table.isRewriteEnabled());
- if (table.getTableType() != null) builder.setTableType(table.getTableType());
- if (table.getPrivileges() != null) {
- builder.setPrivileges(buildPrincipalPrivilegeSet(table.getPrivileges()));
- }
- // Set only if table is temporary
- if (table.isTemporary()) {
- builder.setIsTemporary(table.isTemporary());
- }
- result[1] = builder.build().toByteArray();
- return result;
- }
-
- /**
- * Deserialize a table. This version should be used when the table key is not already
- * known (eg a scan).
- * @param key the key fetched from HBase
- * @param serialized the value fetched from HBase
- * @return A struct that contains the table plus parts of the storage descriptor
- */
- static StorageDescriptorParts deserializeTable(byte[] key, byte[] serialized)
- throws InvalidProtocolBufferException {
- String[] keys = deserializeKey(key);
- return deserializeTable(keys[0], keys[1], serialized);
- }
-
- /**
- * Deserialize a table. This version should be used when the table key is
- * known (eg a get).
- * @param dbName database name
- * @param tableName table name
- * @param serialized the value fetched from HBase
- * @return A struct that contains the partition plus parts of the storage descriptor
- */
- static StorageDescriptorParts deserializeTable(String dbName, String tableName,
- byte[] serialized)
- throws InvalidProtocolBufferException {
- HbaseMetastoreProto.Table proto = HbaseMetastoreProto.Table.parseFrom(serialized);
- Table table = new Table();
- StorageDescriptorParts sdParts = new StorageDescriptorParts();
- sdParts.containingTable = table;
- table.setDbName(dbName);
- table.setTableName(tableName);
- table.setOwner(proto.getOwner());
- table.setCreateTime((int)proto.getCreateTime());
- table.setLastAccessTime((int)proto.getLastAccessTime());
- table.setRetention((int)proto.getRetention());
- if (proto.hasLocation()) sdParts.location = proto.getLocation();
- if (proto.hasSdParameters()) sdParts.parameters = buildParameters(proto.getSdParameters());
- sdParts.sdHash = proto.getSdHash().toByteArray();
- table.setPartitionKeys(convertFieldSchemaListFromProto(proto.getPartitionKeysList()));
- table.setParameters(buildParameters(proto.getParameters()));
- if (proto.hasViewOriginalText()) table.setViewOriginalText(proto.getViewOriginalText());
- if (proto.hasViewExpandedText()) table.setViewExpandedText(proto.getViewExpandedText());
- table.setRewriteEnabled(proto.getIsRewriteEnabled());
- table.setTableType(proto.getTableType());
- if (proto.hasPrivileges()) {
- table.setPrivileges(buildPrincipalPrivilegeSet(proto.getPrivileges()));
- }
- if (proto.hasIsTemporary()) table.setTemporary(proto.getIsTemporary());
- return sdParts;
- }
-
- /**
- * Serialize an index
- * @param index index object
- * @param sdHash hash that is being used as a key for the enclosed storage descriptor
- * @return First element is the key, second is the serialized index
- */
- static byte[][] serializeIndex(Index index, byte[] sdHash) {
- byte[][] result = new byte[2][];
- result[0] = buildKey(HiveStringUtils.normalizeIdentifier(index.getDbName()),
- HiveStringUtils.normalizeIdentifier(index.getOrigTableName()),
- HiveStringUtils.normalizeIdentifier(index.getIndexName()));
- HbaseMetastoreProto.Index.Builder builder = HbaseMetastoreProto.Index.newBuilder();
- builder.setDbName(index.getDbName());
- builder.setOrigTableName(index.getOrigTableName());
- if (index.getSd().getLocation() != null) builder.setLocation(index.getSd().getLocation());
- if (index.getSd().getParameters() != null) {
- builder.setSdParameters(buildParameters(index.getSd().getParameters()));
- }
- if (index.getIndexHandlerClass() != null) {
- builder.setIndexHandlerClass(index.getIndexHandlerClass());
- }
- if (index.getIndexTableName() != null) {
- builder.setIndexTableName(index.getIndexTableName());
- }
- builder
- .setCreateTime(index.getCreateTime())
- .setLastAccessTime(index.getLastAccessTime())
- .setDeferredRebuild(index.isDeferredRebuild());
- if (index.getParameters() != null) {
- builder.setParameters(buildParameters(index.getParameters()));
- }
- if (sdHash != null) {
- builder.setSdHash(ByteString.copyFrom(sdHash));
- }
- result[1] = builder.build().toByteArray();
- return result;
- }
-
- /**
- * Deserialize an index. This version should be used when the index key is not already
- * known (eg a scan).
- * @param key the key fetched from HBase
- * @param serialized the value fetched from HBase
- * @return A struct that contains the index plus parts of the storage descriptor
- */
- static StorageDescriptorParts deserializeIndex(byte[] key, byte[] serialized)
- throws InvalidProtocolBufferException {
- String[] keys = deserializeKey(key);
- return deserializeIndex(keys[0], keys[1], keys[2], serialized);
- }
-
- /**
- * Deserialize an index. This version should be used when the table key is
- * known (eg a get).
- * @param dbName database name
- * @param origTableName original table name
- * @param indexName index name
- * @param serialized the value fetched from HBase
- * @return A struct that contains the index plus parts of the storage descriptor
- */
- static StorageDescriptorParts deserializeIndex(String dbName, String origTableName,
- String indexName, byte[] serialized)
- throws InvalidProtocolBufferException {
- HbaseMetastoreProto.Index proto = HbaseMetastoreProto.Index.parseFrom(serialized);
- Index index = new Index();
- StorageDescriptorParts sdParts = new StorageDescriptorParts();
- sdParts.containingIndex = index;
- index.setDbName(dbName);
- index.setIndexName(indexName);
- index.setOrigTableName(origTableName);
- if (proto.hasLocation()) sdParts.location = proto.getLocation();
- if (proto.hasSdParameters()) sdParts.parameters = buildParameters(proto.getSdParameters());
- if (proto.hasIndexHandlerClass()) {
- index.setIndexHandlerClass(proto.getIndexHandlerClass());
- }
- if (proto.hasIndexTableName()) {
- index.setIndexTableName(proto.getIndexTableName());
- }
- index.setCreateTime(proto.getCreateTime());
- index.setLastAccessTime(proto.getLastAccessTime());
- index.setDeferredRebuild(proto.getDeferredRebuild());
- index.setParameters(buildParameters(proto.getParameters()));
- if (proto.hasSdHash()) {
- sdParts.sdHash = proto.getSdHash().toByteArray();
- }
- return sdParts;
- }
-
- static byte[] serializeBloomFilter(String dbName, String tableName, BloomFilter bloom) {
- long[] bitSet = bloom.getBitSet();
- List<Long> bits = new ArrayList<>(bitSet.length);
- for (int i = 0; i < bitSet.length; i++) bits.add(bitSet[i]);
- HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter protoBloom =
- HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.newBuilder()
- .setNumBits(bloom.getBitSize())
- .setNumFuncs(bloom.getNumHashFunctions())
- .addAllBits(bits)
- .build();
-
- HbaseMetastoreProto.AggrStatsBloomFilter proto =
- HbaseMetastoreProto.AggrStatsBloomFilter.newBuilder()
- .setDbName(ByteString.copyFrom(dbName.getBytes(ENCODING)))
- .setTableName(ByteString.copyFrom(tableName.getBytes(ENCODING)))
- .setBloomFilter(protoBloom)
- .setAggregatedAt(System.currentTimeMillis())
- .build();
-
- return proto.toByteArray();
- }
-
- private static HbaseMetastoreProto.ColumnStats protoBufStatsForOneColumn(
- ColumnStatistics partitionColumnStats, ColumnStatisticsObj colStats) throws IOException {
- HbaseMetastoreProto.ColumnStats.Builder builder = HbaseMetastoreProto.ColumnStats.newBuilder();
- if (partitionColumnStats != null) {
- builder.setLastAnalyzed(partitionColumnStats.getStatsDesc().getLastAnalyzed());
- }
- assert colStats.getColType() != null;
- builder.setColumnType(colStats.getColType());
- assert colStats.getColName() != null;
- builder.setColumnName(colStats.getColName());
-
- ColumnStatisticsData colData = colStats.getStatsData();
- switch (colData.getSetField()) {
- case BOOLEAN_STATS:
- BooleanColumnStatsData boolData = colData.getBooleanStats();
- builder.setNumNulls(boolData.getNumNulls());
- builder.setBoolStats(HbaseMetastoreProto.ColumnStats.BooleanStats.newBuilder()
- .setNumTrues(boolData.getNumTrues()).setNumFalses(boolData.getNumFalses()).build());
- break;
-
- case LONG_STATS:
- LongColumnStatsData longData = colData.getLongStats();
- builder.setNumNulls(longData.getNumNulls());
- builder.setNumDistinctValues(longData.getNumDVs());
- if (longData.isSetBitVectors()) {
- builder.setBitVectors(longData.getBitVectors());
- }
- builder.setLongStats(HbaseMetastoreProto.ColumnStats.LongStats.newBuilder()
- .setLowValue(longData.getLowValue()).setHighValue(longData.getHighValue()).build());
- break;
-
- case DOUBLE_STATS:
- DoubleColumnStatsData doubleData = colData.getDoubleStats();
- builder.setNumNulls(doubleData.getNumNulls());
- builder.setNumDistinctValues(doubleData.getNumDVs());
- if (doubleData.isSetBitVectors()) {
- builder.setBitVectors(doubleData.getBitVectors());
- }
- builder.setDoubleStats(HbaseMetastoreProto.ColumnStats.DoubleStats.newBuilder()
- .setLowValue(doubleData.getLowValue()).setHighValue(doubleData.getHighValue()).build());
- break;
-
- case STRING_STATS:
- StringColumnStatsData stringData = colData.getStringStats();
- builder.setNumNulls(stringData.getNumNulls());
- builder.setNumDistinctValues(stringData.getNumDVs());
- if (stringData.isSetBitVectors()) {
- builder.setBitVectors(stringData.getBitVectors());
- }
- builder.setStringStats(HbaseMetastoreProto.ColumnStats.StringStats.newBuilder()
- .setMaxColLength(stringData.getMaxColLen()).setAvgColLength(stringData.getAvgColLen())
- .build());
- break;
-
- case BINARY_STATS:
- BinaryColumnStatsData binaryData = colData.getBinaryStats();
- builder.setNumNulls(binaryData.getNumNulls());
- builder.setBinaryStats(HbaseMetastoreProto.ColumnStats.StringStats.newBuilder()
- .setMaxColLength(binaryData.getMaxColLen()).setAvgColLength(binaryData.getAvgColLen())
- .build());
- break;
-
- case DECIMAL_STATS:
- DecimalColumnStatsData decimalData = colData.getDecimalStats();
- builder.setNumNulls(decimalData.getNumNulls());
- builder.setNumDistinctValues(decimalData.getNumDVs());
- if (decimalData.isSetBitVectors()) {
- builder.setBitVectors(decimalData.getBitVectors());
- }
- if (decimalData.getLowValue() != null && decimalData.getHighValue() != null) {
- builder.setDecimalStats(
- HbaseMetastoreProto.ColumnStats.DecimalStats
- .newBuilder()
- .setLowValue(
- HbaseMetastoreProto.ColumnStats.DecimalStats.Decimal.newBuilder()
- .setUnscaled(ByteString.copyFrom(decimalData.getLowValue().getUnscaled()))
- .setScale(decimalData.getLowValue().getScale()).build())
- .setHighValue(
- HbaseMetastoreProto.ColumnStats.DecimalStats.Decimal.newBuilder()
- .setUnscaled(ByteString.copyFrom(decimalData.getHighValue().getUnscaled()))
- .setScale(decimalData.getHighValue().getScale()).build())).build();
- } else {
- builder.setDecimalStats(HbaseMetastoreProto.ColumnStats.DecimalStats.newBuilder().clear()
- .build());
- }
- break;
-
- default:
- throw new RuntimeException("Woh, bad. Unknown stats type!");
- }
- return builder.build();
- }
-
- static byte[] serializeStatsForOneColumn(ColumnStatistics partitionColumnStats,
- ColumnStatisticsObj colStats) throws IOException {
- return protoBufStatsForOneColumn(partitionColumnStats, colStats).toByteArray();
- }
-
- static ColumnStatisticsObj deserializeStatsForOneColumn(ColumnStatistics partitionColumnStats,
- byte[] bytes) throws IOException {
- HbaseMetastoreProto.ColumnStats proto = HbaseMetastoreProto.ColumnStats.parseFrom(bytes);
- return statsForOneColumnFromProtoBuf(partitionColumnStats, proto);
- }
-
- private static ColumnStatisticsObj
- statsForOneColumnFromProtoBuf(ColumnStatistics partitionColumnStats,
- HbaseMetastoreProto.ColumnStats proto) throws IOException {
- ColumnStatisticsObj colStats = new ColumnStatisticsObj();
- long lastAnalyzed = proto.getLastAnalyzed();
- if (partitionColumnStats != null) {
- partitionColumnStats.getStatsDesc().setLastAnalyzed(
- Math.max(lastAnalyzed, partitionColumnStats.getStatsDesc().getLastAnalyzed()));
- }
- colStats.setColType(proto.getColumnType());
- colStats.setColName(proto.getColumnName());
-
- ColumnStatisticsData colData = new ColumnStatisticsData();
- if (proto.hasBoolStats()) {
- BooleanColumnStatsData boolData = new BooleanColumnStatsData();
- boolData.setNumTrues(proto.getBoolStats().getNumTrues());
- boolData.setNumFalses(proto.getBoolStats().getNumFalses());
- boolData.setNumNulls(proto.getNumNulls());
- colData.setBooleanStats(boolData);
- } else if (proto.hasLongStats()) {
- LongColumnStatsData longData = new LongColumnStatsData();
- if (proto.getLongStats().hasLowValue()) {
- longData.setLowValue(proto.getLongStats().getLowValue());
- }
- if (proto.getLongStats().hasHighValue()) {
- longData.setHighValue(proto.getLongStats().getHighValue());
- }
- longData.setNumNulls(proto.getNumNulls());
- longData.setNumDVs(proto.getNumDistinctValues());
- longData.setBitVectors(proto.getBitVectors());
- colData.setLongStats(longData);
- } else if (proto.hasDoubleStats()) {
- DoubleColumnStatsData doubleData = new DoubleColumnStatsData();
- if (proto.getDoubleStats().hasLowValue()) {
- doubleData.setLowValue(proto.getDoubleStats().getLowValue());
- }
- if (proto.getDoubleStats().hasHighValue()) {
- doubleData.setHighValue(proto.getDoubleStats().getHighValue());
- }
- doubleData.setNumNulls(proto.getNumNulls());
- doubleData.setNumDVs(proto.getNumDistinctValues());
- doubleData.setBitVectors(proto.getBitVectors());
- colData.setDoubleStats(doubleData);
- } else if (proto.hasStringStats()) {
- StringColumnStatsData stringData = new StringColumnStatsData();
- stringData.setMaxColLen(proto.getStringStats().getMaxColLength());
- stringData.setAvgColLen(proto.getStringStats().getAvgColLength());
- stringData.setNumNulls(proto.getNumNulls());
- stringData.setNumDVs(proto.getNumDistinctValues());
- stringData.setBitVectors(proto.getBitVectors());
- colData.setStringStats(stringData);
- } else if (proto.hasBinaryStats()) {
- BinaryColumnStatsData binaryData = new BinaryColumnStatsData();
- binaryData.setMaxColLen(proto.getBinaryStats().getMaxColLength());
- binaryData.setAvgColLen(proto.getBinaryStats().getAvgColLength());
- binaryData.setNumNulls(proto.getNumNulls());
- colData.setBinaryStats(binaryData);
- } else if (proto.hasDecimalStats()) {
- DecimalColumnStatsData decimalData = new DecimalColumnStatsData();
- if (proto.getDecimalStats().hasHighValue()) {
- Decimal hiVal = new Decimal();
- hiVal.setUnscaled(proto.getDecimalStats().getHighValue().getUnscaled().toByteArray());
- hiVal.setScale((short) proto.getDecimalStats().getHighValue().getScale());
- decimalData.setHighValue(hiVal);
- }
- if (proto.getDecimalStats().hasLowValue()) {
- Decimal loVal = new Decimal();
- loVal.setUnscaled(proto.getDecimalStats().getLowValue().getUnscaled().toByteArray());
- loVal.setScale((short) proto.getDecimalStats().getLowValue().getScale());
- decimalData.setLowValue(loVal);
- }
- decimalData.setNumNulls(proto.getNumNulls());
- decimalData.setNumDVs(proto.getNumDistinctValues());
- decimalData.setBitVectors(proto.getBitVectors());
- colData.setDecimalStats(decimalData);
- } else {
- throw new RuntimeException("Woh, bad. Unknown stats type!");
- }
- colStats.setStatsData(colData);
- return colStats;
- }
-
- static byte[] serializeAggrStats(AggrStats aggrStats) throws IOException {
- List<HbaseMetastoreProto.ColumnStats> protoColStats =
- new ArrayList<>(aggrStats.getColStatsSize());
- for (ColumnStatisticsObj cso : aggrStats.getColStats()) {
- protoColStats.add(protoBufStatsForOneColumn(null, cso));
- }
- return HbaseMetastoreProto.AggrStats.newBuilder()
- .setPartsFound(aggrStats.getPartsFound())
- .addAllColStats(protoColStats)
- .build()
- .toByteArray();
- }
-
- static AggrStats deserializeAggrStats(byte[] serialized) throws IOException {
- HbaseMetastoreProto.AggrStats protoAggrStats =
- HbaseMetastoreProto.AggrStats.parseFrom(serialized);
- AggrStats aggrStats = new AggrStats();
- aggrStats.setPartsFound(protoAggrStats.getPartsFound());
- for (HbaseMetastoreProto.ColumnStats protoCS : protoAggrStats.getColStatsList()) {
- aggrStats.addToColStats(statsForOneColumnFromProtoBuf(null, protoCS));
- }
- return aggrStats;
- }
-
- /**
- * Serialize a delegation token
- * @param tokenIdentifier
- * @param delegationToken
- * @return two byte arrays, first contains the key, the second the serialized value.
- */
- static byte[][] serializeDelegationToken(String tokenIdentifier, String delegationToken) {
- byte[][] result = new byte[2][];
- result[0] = buildKey(tokenIdentifier);
- result[1] = HbaseMetastoreProto.DelegationToken.newBuilder()
- .setTokenStr(delegationToken)
- .build()
- .toByteArray();
- return result;
- }
-
- /**
- * Deserialize a delegation token.
- * @param value value fetched from hbase
- * @return A delegation token.
- * @throws InvalidProtocolBufferException
- */
- static String deserializeDelegationToken(byte[] value) throws InvalidProtocolBufferException {
- HbaseMetastoreProto.DelegationToken protoToken =
- HbaseMetastoreProto.DelegationToken.parseFrom(value);
- return protoToken.getTokenStr();
- }
-
- /**
- * Serialize a master key
- * @param seqNo
- * @param key
- * @return two byte arrays, first contains the key, the second the serialized value.
- */
- static byte[][] serializeMasterKey(Integer seqNo, String key) {
- byte[][] result = new byte[2][];
- result[0] = buildKey(seqNo.toString());
- result[1] = HbaseMetastoreProto.MasterKey.newBuilder()
- .setMasterKey(key)
- .build()
- .toByteArray();
- return result;
- }
-
- /**
- * Deserialize a master key.
- * @param value value fetched from hbase
- * @return A master key
- * @throws InvalidProtocolBufferException
- */
- static String deserializeMasterKey(byte[] value) throws InvalidProtocolBufferException {
- HbaseMetastoreProto.MasterKey protoKey = HbaseMetastoreProto.MasterKey.parseFrom(value);
- return protoKey.getMasterKey();
- }
-
- /**
- * Serialize the primary key for a table.
- * @param pk Primary key columns. It is expected that all of these match to one pk, since
- * anything else is meaningless.
- * @return two byte arrays, first containts the hbase key, the second the serialized value.
- */
- static byte[][] serializePrimaryKey(List<SQLPrimaryKey> pk) {
- // First, figure out the dbName and tableName. We expect this to match for all list entries.
- byte[][] result = new byte[2][];
- String dbName = pk.get(0).getTable_db();
- String tableName = pk.get(0).getTable_name();
- result[0] = buildKey(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName));
-
- HbaseMetastoreProto.PrimaryKey.Builder builder = HbaseMetastoreProto.PrimaryKey.newBuilder();
- // Encode the primary key, if present
- builder.setPkName(pk.get(0).getPk_name());
- builder.setEnableConstraint(pk.get(0).isEnable_cstr());
- builder.setValidateConstraint(pk.get(0).isValidate_cstr());
- builder.setRelyConstraint(pk.get(0).isRely_cstr());
-
- for (SQLPrimaryKey pkcol : pk) {
- HbaseMetastoreProto.PrimaryKey.PrimaryKeyColumn.Builder pkColBuilder =
- HbaseMetastoreProto.PrimaryKey.PrimaryKeyColumn.newBuilder();
- pkColBuilder.setColumnName(pkcol.getColumn_name());
- pkColBuilder.setKeySeq(pkcol.getKey_seq());
- builder.addCols(pkColBuilder);
- }
-
- result[1] = builder.build().toByteArray();
- return result;
- }
-
- /**
- * Serialize the foreign key(s) for a table.
- * @param fks Foreign key columns. These may belong to multiple foreign keys.
- * @return two byte arrays, first containts the key, the second the serialized value.
- */
- static byte[][] serializeForeignKeys(List<SQLForeignKey> fks) {
- // First, figure out the dbName and tableName. We expect this to match for all list entries.
- byte[][] result = new byte[2][];
- String dbName = fks.get(0).getFktable_db();
- String tableName = fks.get(0).getFktable_name();
- result[0] = buildKey(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName));
-
- HbaseMetastoreProto.ForeignKeys.Builder builder = HbaseMetastoreProto.ForeignKeys.newBuilder();
-
- // Encode any foreign keys we find. This can be complex because there may be more than
- // one foreign key in here, so we need to detect that.
- Map<String, HbaseMetastoreProto.ForeignKeys.ForeignKey.Builder> fkBuilders = new HashMap<>();
-
- for (SQLForeignKey fkcol : fks) {
- HbaseMetastoreProto.ForeignKeys.ForeignKey.Builder fkBuilder =
- fkBuilders.get(fkcol.getFk_name());
- if (fkBuilder == null) {
- // We haven't seen this key before, so add it
- fkBuilder = HbaseMetastoreProto.ForeignKeys.ForeignKey.newBuilder();
- fkBuilder.setFkName(fkcol.getFk_name());
- fkBuilder.setReferencedDbName(fkcol.getPktable_db());
- assert dbName.equals(fkcol.getFktable_db()) : "You switched databases on me!";
- fkBuilder.setReferencedTableName(fkcol.getPktable_name());
- assert tableName.equals(fkcol.getFktable_name()) : "You switched tables on me!";
- fkBuilder.setReferencedPkName(fkcol.getPk_name());
- fkBuilder.setUpdateRule(fkcol.getUpdate_rule());
- fkBuilder.setDeleteRule(fkcol.getDelete_rule());
- fkBuilder.setEnableConstraint(fkcol.isEnable_cstr());
- fkBuilder.setValidateConstraint(fkcol.isValidate_cstr());
- fkBuilder.setRelyConstraint(fkcol.isRely_cstr());
- fkBuilders.put(fkcol.getFk_name(), fkBuilder);
- }
- HbaseMetastoreProto.ForeignKeys.ForeignKey.ForeignKeyColumn.Builder fkColBuilder =
- HbaseMetastoreProto.ForeignKeys.ForeignKey.ForeignKeyColumn.newBuilder();
- fkColBuilder.setColumnName(fkcol.getFkcolumn_name());
- fkColBuilder.setReferencedColumnName(fkcol.getPkcolumn_name());
- fkColBuilder.setKeySeq(fkcol.getKey_seq());
- fkBuilder.addCols(fkColBuilder);
- }
- for (HbaseMetastoreProto.ForeignKeys.ForeignKey.Builder fkBuilder : fkBuilders.values()) {
- builder.addFks(fkBuilder);
- }
- result[1] = builder.build().toByteArray();
- return result;
- }
-
- /**
- * Serialize the unique constraint(s) for a table.
- * @param uks Unique constraint columns. These may belong to multiple unique constraints.
- * @return two byte arrays, first contains the key, the second the serialized value.
- */
- static byte[][] serializeUniqueConstraints(List<SQLUniqueConstraint> uks) {
- // First, figure out the dbName and tableName. We expect this to match for all list entries.
- byte[][] result = new byte[2][];
- String dbName = uks.get(0).getTable_db();
- String tableName = uks.get(0).getTable_name();
- result[0] = buildKey(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName));
-
- HbaseMetastoreProto.UniqueConstraints.Builder builder =
- HbaseMetastoreProto.UniqueConstraints.newBuilder();
-
- // Encode any foreign keys we find. This can be complex because there may be more than
- // one foreign key in here, so we need to detect that.
- Map<String, HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.Builder> ukBuilders = new HashMap<>();
-
- for (SQLUniqueConstraint ukcol : uks) {
- HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.Builder ukBuilder =
- ukBuilders.get(ukcol.getUk_name());
- if (ukBuilder == null) {
- // We haven't seen this key before, so add it
- ukBuilder = HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.newBuilder();
- ukBuilder.setUkName(ukcol.getUk_name());
- ukBuilder.setEnableConstraint(ukcol.isEnable_cstr());
- ukBuilder.setValidateConstraint(ukcol.isValidate_cstr());
- ukBuilder.setRelyConstraint(ukcol.isRely_cstr());
- ukBuilders.put(ukcol.getUk_name(), ukBuilder);
- }
- assert dbName.equals(ukcol.getTable_db()) : "You switched databases on me!";
- assert tableName.equals(ukcol.getTable_name()) : "You switched tables on me!";
- HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.UniqueConstraintColumn.Builder ukColBuilder =
- HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.UniqueConstraintColumn.newBuilder();
- ukColBuilder.setColumnName(ukcol.getColumn_name());
- ukColBuilder.setKeySeq(ukcol.getKey_seq());
- ukBuilder.addCols(ukColBuilder);
- }
- for (HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.Builder ukBuilder : ukBuilders.values()) {
- builder.addUks(ukBuilder);
- }
- result[1] = builder.build().toByteArray();
- return result;
- }
-
- /**
- * Serialize the not null constraint(s) for a table.
- * @param nns Not null constraint columns. These may belong to multiple constraints.
- * @return two byte arrays, first contains the constraint, the second the serialized value.
- */
- static byte[][] serializeNotNullConstraints(List<SQLNotNullConstraint> nns) {
- // First, figure out the dbName and tableName. We expect this to match for all list entries.
- byte[][] result = new byte[2][];
- String dbName = nns.get(0).getTable_db();
- String tableName = nns.get(0).getTable_name();
- result[0] = buildKey(HiveStringUtils.normalizeIdentifier(dbName),
- HiveStringUtils.normalizeIdentifier(tableName));
-
- HbaseMetastoreProto.NotNullConstraints.Builder builder =
- HbaseMetastoreProto.NotNullConstraints.newBuilder();
-
- // Encode any foreign keys we find. This can be complex because there may be more than
- // one foreign key in here, so we need to detect that.
- Map<String, HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.Builder> nnBuilders = new HashMap<>();
-
- for (SQLNotNullConstraint nncol : nns) {
- HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.Builder nnBuilder =
- nnBuilders.get(nncol.getNn_name());
- if (nnBuilder == null) {
- // We haven't seen this key before, so add it
- nnBuilder = HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.newBuilder();
- nnBuilder.setNnName(nncol.getNn_name());
- nnBuilder.setEnableConstraint(nncol.isEnable_cstr());
- nnBuilder.setValidateConstraint(nncol.isValidate_cstr());
- nnBuilder.setRelyConstraint(nncol.isRely_cstr());
- nnBuilders.put(nncol.getNn_name(), nnBuilder);
- }
- assert dbName.equals(nncol.getTable_db()) : "You switched databases on me!";
- assert tableName.equals(nncol.getTable_name()) : "You switched tables on me!";
- HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.NotNullConstraintColumn.Builder nnColBuilder =
- HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.NotNullConstraintColumn.newBuilder();
- nnColBuilder.setColumnName(nncol.getColumn_name());
- nnBuilder.addCols(nnColBuilder);
- }
- for (HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.Builder nnBuilder : nnBuilders.values()) {
- builder.addNns(nnBuilder);
- }
- result[1] = builder.build().toByteArray();
- return result;
- }
-
- static List<SQLPrimaryKey> deserializePrimaryKey(String dbName, String tableName, byte[] value)
- throws InvalidProtocolBufferException {
- HbaseMetastoreProto.PrimaryKey proto = HbaseMetastoreProto.PrimaryKey.parseFrom(value);
- List<SQLPrimaryKey> result = new ArrayList<>();
- for (HbaseMetastoreProto.PrimaryKey.PrimaryKeyColumn protoPkCol : proto.getColsList()) {
- result.add(new SQLPrimaryKey(dbName, tableName, protoPkCol.getColumnName(),
- protoPkCol.getKeySeq(), proto.getPkName(), proto.getEnableConstraint(),
- proto.getValidateConstraint(), proto.getRelyConstraint()));
- }
-
- return result;
- }
-
- static List<SQLUniqueConstraint> deserializeUniqueConstraint(String dbName, String tableName, byte[] value)
- throws InvalidProtocolBufferException {
- List<SQLUniqueConstraint> result = new ArrayList<>();
- HbaseMetastoreProto.UniqueConstraints protoConstraints =
- HbaseMetastoreProto.UniqueConstraints.parseFrom(value);
-
- for (HbaseMetastoreProto.UniqueConstraints.UniqueConstraint proto : protoConstraints.getUksList()) {
- for (HbaseMetastoreProto.UniqueConstraints.UniqueConstraint.UniqueConstraintColumn protoUkCol :
- proto.getColsList()) {
- result.add(new SQLUniqueConstraint(dbName, tableName, protoUkCol.getColumnName(),
- protoUkCol.getKeySeq(),
- proto.getUkName(), proto.getEnableConstraint(),
- proto.getValidateConstraint(), proto.getRelyConstraint()));
- }
- }
- return result;
- }
-
- static List<SQLNotNullConstraint> deserializeNotNullConstraint(String dbName, String tableName, byte[] value)
- throws InvalidProtocolBufferException {
- List<SQLNotNullConstraint> result = new ArrayList<>();
- HbaseMetastoreProto.NotNullConstraints protoConstraints =
- HbaseMetastoreProto.NotNullConstraints.parseFrom(value);
-
- for (HbaseMetastoreProto.NotNullConstraints.NotNullConstraint proto : protoConstraints.getNnsList()) {
- for (HbaseMetastoreProto.NotNullConstraints.NotNullConstraint.NotNullConstraintColumn protoNnCol :
- proto.getColsList()) {
- result.add(new SQLNotNullConstraint(dbName, tableName, protoNnCol.getColumnName(),
- proto.getNnName(), proto.getEnableConstraint(),
- proto.getValidateConstraint(), proto.getRelyConstraint()));
- }
- }
- return result;
- }
-
- static List<SQLForeignKey> deserializeForeignKeys(String dbName, String tableName, byte[] value)
- throws InvalidProtocolBufferException {
- List<SQLForeignKey> result = new ArrayList<>();
- HbaseMetastoreProto.ForeignKeys protoConstraints =
- HbaseMetastoreProto.ForeignKeys.parseFrom(value);
-
- for (HbaseMetastoreProto.ForeignKeys.ForeignKey protoFk : protoConstraints.getFksList()) {
- for (HbaseMetastoreProto.ForeignKeys.ForeignKey.ForeignKeyColumn protoFkCol :
- protoFk.getColsList()) {
- result.add(new SQLForeignKey(protoFk.getReferencedDbName(), protoFk.getReferencedTableName(),
- protoFkCol.getReferencedColumnName(), dbName, tableName, protoFkCol.getColumnName(),
- protoFkCol.getKeySeq(), protoFk.getUpdateRule(), protoFk.getDeleteRule(),
- protoFk.getFkName(), protoFk.getReferencedPkName(), protoFk.getEnableConstraint(),
- protoFk.getValidateConstraint(), protoFk.getRelyConstraint()));
- }
- }
- return result;
- }
-
- /**
- * @param keyStart byte array representing the start prefix
- * @return byte array corresponding to the next possible prefix
- */
- static byte[] getEndPrefix(byte[] keyStart) {
- if (keyStart == null) {
- return null;
- }
- // Since this is a prefix and not full key, the usual hbase technique of
- // appending 0 byte does not work. Instead of that, increment the last byte.
- byte[] keyEnd = Arrays.copyOf(keyStart, keyStart.length);
- keyEnd[keyEnd.length - 1]++;
- return keyEnd;
- }
-
- static byte[] makeLongKey(long v) {
- byte[] b = new byte[8];
- b[0] = (byte)(v >>> 56);
- b[1] = (byte)(v >>> 48);
- b[2] = (byte)(v >>> 40);
- b[3] = (byte)(v >>> 32);
- b[4] = (byte)(v >>> 24);
- b[5] = (byte)(v >>> 16);
- b[6] = (byte)(v >>> 8);
- b[7] = (byte)(v >>> 0);
- return b;
- }
-
- public static double getDoubleValue(Decimal decimal) {
- return new BigDecimal(new BigInteger(decimal.getUnscaled()), decimal.getScale()).doubleValue();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/ObjectCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/ObjectCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/ObjectCache.java
deleted file mode 100644
index 5bbed5d..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/ObjectCache.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A generic class for caching objects obtained from HBase. Currently a set of
- * convenience methods around a {@link java.util.HashMap} with a max size but built
- * as a separate class in case we want to switch out the implementation to something more
- * efficient. The cache has a max size; when this is exceeded any additional entries are dropped
- * on the floor.
- *
- * This cache is local to a particular thread and thus is not synchronized. It is intended to be
- * flushed before a query begins to make sure it doesn't carry old versions of objects between
- * queries (that is, an object may have changed between two queries, we want to get the newest
- * version).
- */
-class ObjectCache<K, V> {
- private Map<K, V> cache;
- private final int maxSize;
- private Counter hits;
- private Counter misses;
- private Counter overflows;
-
- /**
- *
- * @param max maximum number of objects to store in the cache. When max is reached, eviction
- * policy is MRU.
- * @param hits counter to increment when we find an element in the cache
- * @param misses counter to increment when we do not find an element in the cache
- * @param overflows counter to increment when we do not have room for an element in the cache
- */
- ObjectCache(int max, Counter hits, Counter misses, Counter overflows) {
- maxSize = max;
- cache = new HashMap<K, V>();
- this.hits = hits;
- this.misses = misses;
- this.overflows = overflows;
- }
-
- void put(K key, V value) {
- if (cache.size() < maxSize) {
- cache.put(key, value);
- } else {
- overflows.incr();
- }
- }
-
- V get(K key) {
- V val = cache.get(key);
- if (val == null) misses.incr();
- else hits.incr();
- return val;
- }
-
- void remove(K key) {
- cache.remove(key);
- }
-
- void flush() {
- cache.clear();
- }
-}
[14/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
deleted file mode 100644
index c29e46a..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Integration tests with HBase Mini-cluster for HBaseStore
- */
-public class TestStorageDescriptorSharing extends HBaseIntegrationTests {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreIntegration.class.getName());
-
- private MessageDigest md;
-
- @BeforeClass
- public static void startup() throws Exception {
- HBaseIntegrationTests.startMiniCluster();
- }
-
- @AfterClass
- public static void shutdown() throws Exception {
- HBaseIntegrationTests.shutdownMiniCluster();
- }
-
- @Before
- public void setup() throws IOException {
- setupConnection();
- setupHBaseStore();
- try {
- md = MessageDigest.getInstance("MD5");
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Test
- public void createManyPartitions() throws Exception {
- String dbName = "default";
- String tableName = "manyParts";
- int startTime = (int)(System.currentTimeMillis() / 1000);
- List<FieldSchema> cols = new ArrayList<FieldSchema>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- List<FieldSchema> partCols = new ArrayList<FieldSchema>();
- partCols.add(new FieldSchema("pc", "string", ""));
- Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
- for (String val : partVals) {
- List<String> vals = new ArrayList<String>();
- vals.add(val);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/pc=" + val);
- Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
- emptyParameters);
- store.addPartition(part);
-
- Partition p = store.getPartition(dbName, tableName, vals);
- Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
- }
-
- Assert.assertEquals(1, HBaseReadWrite.getInstance().countStorageDescriptor());
-
- String tableName2 = "differentTable";
- sd = new StorageDescriptor(cols, "file:/tmp", "input2", "output", false, 0,
- serde, null, null, emptyParameters);
- table = new Table(tableName2, "default", "me", startTime, startTime, 0, sd, null,
- emptyParameters, null, null, null);
- store.createTable(table);
-
- Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
-
- // Drop one of the partitions and make sure it doesn't drop the storage descriptor
- store.dropPartition(dbName, tableName, Arrays.asList(partVals.get(0)));
- Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
-
- // Alter the second table in a few ways to make sure it changes it's descriptor properly
- table = store.getTable(dbName, tableName2);
- byte[] sdHash = HBaseUtils.hashStorageDescriptor(table.getSd(), md);
-
- // Alter the table without touching the storage descriptor
- table.setLastAccessTime(startTime + 1);
- store.alterTable(dbName, tableName2, table);
- Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
- table = store.getTable(dbName, tableName2);
- byte[] alteredHash = HBaseUtils.hashStorageDescriptor(table.getSd(), md);
- Assert.assertArrayEquals(sdHash, alteredHash);
-
- // Alter the table, changing the storage descriptor
- table.getSd().setOutputFormat("output_changed");
- store.alterTable(dbName, tableName2, table);
- Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
- table = store.getTable(dbName, tableName2);
- alteredHash = HBaseUtils.hashStorageDescriptor(table.getSd(), md);
- Assert.assertFalse(Arrays.equals(sdHash, alteredHash));
-
- // Alter one of the partitions without touching the storage descriptor
- Partition part = store.getPartition(dbName, tableName, Arrays.asList(partVals.get(1)));
- sdHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
- part.setLastAccessTime(part.getLastAccessTime() + 1);
- store.alterPartition(dbName, tableName, Arrays.asList(partVals.get(1)), part);
- Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
- part = store.getPartition(dbName, tableName, Arrays.asList(partVals.get(1)));
- alteredHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
- Assert.assertArrayEquals(sdHash, alteredHash);
-
- // Alter the partition, changing the storage descriptor
- part.getSd().setOutputFormat("output_changed_some_more");
- store.alterPartition(dbName, tableName, Arrays.asList(partVals.get(1)), part);
- Assert.assertEquals(3, HBaseReadWrite.getInstance().countStorageDescriptor());
- part = store.getPartition(dbName, tableName, Arrays.asList(partVals.get(1)));
- alteredHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
- Assert.assertFalse(Arrays.equals(sdHash, alteredHash));
-
- // Alter multiple partitions without touching the storage descriptors
- List<Partition> parts = store.getPartitions(dbName, tableName, -1);
- sdHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
- for (int i = 1; i < 3; i++) {
- parts.get(i).setLastAccessTime(97);
- }
- List<List<String>> listPartVals = new ArrayList<List<String>>();
- for (String pv : partVals.subList(1, partVals.size())) {
- listPartVals.add(Arrays.asList(pv));
- }
- store.alterPartitions(dbName, tableName, listPartVals, parts);
- Assert.assertEquals(3, HBaseReadWrite.getInstance().countStorageDescriptor());
- parts = store.getPartitions(dbName, tableName, -1);
- alteredHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
- Assert.assertArrayEquals(sdHash, alteredHash);
-
- // Alter multiple partitions changning the storage descriptors
- parts = store.getPartitions(dbName, tableName, -1);
- sdHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
- for (int i = 1; i < 3; i++) {
- parts.get(i).getSd().setOutputFormat("yet_a_different_of");
- }
- store.alterPartitions(dbName, tableName, listPartVals, parts);
- Assert.assertEquals(4, HBaseReadWrite.getInstance().countStorageDescriptor());
- parts = store.getPartitions(dbName, tableName, -1);
- alteredHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
- Assert.assertFalse(Arrays.equals(sdHash, alteredHash));
-
- for (String partVal : partVals.subList(1, partVals.size())) {
- store.dropPartition(dbName, tableName, Arrays.asList(partVal));
- }
- store.dropTable(dbName, tableName);
- store.dropTable(dbName, tableName2);
-
- Assert.assertEquals(0, HBaseReadWrite.getInstance().countStorageDescriptor());
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index c17ca10..dd2be96 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -90,7 +90,7 @@ public class TestLocationQueries extends BaseTestQueries {
String hadoopVer, String locationSubdir)
throws Exception
{
- super(outDir, logDir, miniMr, null, hadoopVer, "", "", false, false);
+ super(outDir, logDir, miniMr, null, hadoopVer, "", "", false);
this.locationSubdir = locationSubdir;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
index 749abb5..a4dd07e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
@@ -29,7 +29,7 @@ public class AccumuloQTestUtil extends QTestUtil {
public AccumuloQTestUtil(String outDir, String logDir, MiniClusterType miniMr,
AccumuloTestSetup setup, String initScript, String cleanupScript) throws Exception {
- super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false, false);
+ super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false);
setup.setupWithHiveConf(conf);
this.setup = setup;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
index c12f51e..72336e2 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
@@ -45,7 +45,7 @@ public abstract class AbstractCliConfig {
public static final String HIVE_ROOT = getHiveRoot();
public static enum MetastoreType {
- sql, hbase
+ sql
};
private MetastoreType metastoreType = MetastoreType.sql;
@@ -413,8 +413,6 @@ public abstract class AbstractCliConfig {
if (metaStoreTypeProperty != null) {
if (metaStoreTypeProperty.equalsIgnoreCase("sql")) {
metastoreType = MetastoreType.sql;
- } else if (metaStoreTypeProperty.equalsIgnoreCase("hbase")) {
- metastoreType = MetastoreType.hbase;
} else {
throw new IllegalArgumentException("Unknown metastore type: " + metaStoreTypeProperty);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 02abe53..67e03a4 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -55,11 +55,10 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
String hiveConfDir = cliConfig.getHiveConfDir();
String initScript = cliConfig.getInitScript();
String cleanupScript = cliConfig.getCleanupScript();
- boolean useHBaseMetastore = cliConfig.getMetastoreType() == MetastoreType.hbase;
try {
String hadoopVer = cliConfig.getHadoopVersion();
qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, true);
if (Strings.isNullOrEmpty(qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH))) {
fail(String.format("%s must be set. Try setting in blobstore-conf.xml", HCONF_TEST_BLOBSTORE_PATH));
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index d59b650..a1762ec 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -55,7 +55,6 @@ public class CoreCliDriver extends CliAdapter {
final String hiveConfDir = cliConfig.getHiveConfDir();
final String initScript = cliConfig.getInitScript();
final String cleanupScript = cliConfig.getCleanupScript();
- final boolean useHBaseMetastore = cliConfig.getMetastoreType() == MetastoreType.hbase;
try {
final String hadoopVer = cliConfig.getHadoopVersion();
@@ -63,8 +62,7 @@ public class CoreCliDriver extends CliAdapter {
@Override
public QTestUtil invokeInternal() throws Exception {
return new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true,
- cliConfig.getFsType());
+ hiveConfDir, hadoopVer, initScript, cleanupScript, true, cliConfig.getFsType());
}
}.invoke("QtestUtil instance created", LOG, true);
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
index bff81dd..64b419b 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
@@ -52,7 +52,7 @@ public class CoreCompareCliDriver extends CliAdapter{
try {
String hadoopVer = cliConfig.getHadoopVersion();
qt = new QTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript, false, false);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, false);
// do a one time initialization
qt.cleanUp();
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 438a61e..7947988 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -45,7 +45,7 @@ public class CoreNegativeCliDriver extends CliAdapter{
try {
String hadoopVer = cliConfig.getHadoopVersion();
qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR,
- hiveConfDir, hadoopVer, initScript, cleanupScript, false, false);
+ hiveConfDir, hadoopVer, initScript, cleanupScript, false);
// do a one time initialization
qt.cleanUp();
qt.createSources();
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index 34eeb77..d80bd44 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -61,7 +61,7 @@ public class CorePerfCliDriver extends CliAdapter{
String hadoopVer = cliConfig.getHadoopVersion();
qt = new QTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, hiveConfDir,
hadoopVer, initScript,
- cleanupScript, false, false);
+ cleanupScript, false);
// do a one time initialization
qt.cleanUp();
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
index 01faaba..aeb7215 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
@@ -46,7 +46,7 @@ public class HBaseQTestUtil extends QTestUtil {
String initScript, String cleanupScript)
throws Exception {
- super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false, false);
+ super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false);
hbaseSetup = setup;
hbaseSetup.preTest(conf);
this.conn = setup.getConnection();
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
deleted file mode 100644
index 21e8f7e..0000000
--- a/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-import java.util.List;
-
-public class HBaseStoreTestUtil {
- public static void initHBaseMetastore(HBaseAdmin admin, HiveConf conf) throws Exception {
- for (String tableName : HBaseReadWrite.tableNames) {
- List<byte[]> families = HBaseReadWrite.columnFamilies.get(tableName);
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
- for (byte[] family : families) {
- HColumnDescriptor columnDesc = new HColumnDescriptor(family);
- desc.addFamily(columnDesc);
- }
- admin.createTable(desc);
- }
- admin.close();
- if (conf != null) {
- HBaseReadWrite.setConf(conf);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 535cfd9..825f826 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -103,7 +103,6 @@ import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.hbase.HBaseStore;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -202,7 +201,6 @@ public class QTestUtil {
private final String initScript;
private final String cleanupScript;
- private boolean useHBaseMetastore = false;
public interface SuiteAddTestFunctor {
public void addTestToSuite(TestSuite suite, Object setup, String tName);
@@ -347,14 +345,9 @@ public class QTestUtil {
conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, true);
}
- if (!useHBaseMetastore) {
- // Plug verifying metastore in for testing DirectSQL.
- conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
- "org.apache.hadoop.hive.metastore.VerifyingObjectStore");
- } else {
- conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, HBaseStore.class.getName());
- conf.setBoolVar(ConfVars.METASTORE_FASTPATH, true);
- }
+ // Plug verifying metastore in for testing DirectSQL.
+ conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL,
+ "org.apache.hadoop.hive.metastore.VerifyingObjectStore");
if (mr != null) {
mr.setupConfiguration(conf);
@@ -514,40 +507,22 @@ public class QTestUtil {
return "jceks://file" + new Path(keyDir, "test.jks").toUri();
}
- private void startMiniHBaseCluster() throws Exception {
- Configuration hbaseConf = HBaseConfiguration.create();
- hbaseConf.setInt("hbase.master.info.port", -1);
- utility = new HBaseTestingUtility(hbaseConf);
- utility.startMiniCluster();
- conf = new HiveConf(utility.getConfiguration(), Driver.class);
- HBaseAdmin admin = utility.getHBaseAdmin();
- // Need to use reflection here to make compilation pass since HBaseIntegrationTests
- // is not compiled in hadoop-1. All HBaseMetastore tests run under hadoop-2, so this
- // guarantee HBaseIntegrationTests exist when we hitting this code path
- java.lang.reflect.Method initHBaseMetastoreMethod = Class.forName(
- "org.apache.hadoop.hive.metastore.hbase.HBaseStoreTestUtil")
- .getMethod("initHBaseMetastore", HBaseAdmin.class, HiveConf.class);
- initHBaseMetastoreMethod.invoke(null, admin, conf);
- conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, HBaseStore.class.getName());
- conf.setBoolVar(ConfVars.METASTORE_FASTPATH, true);
- }
-
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String confDir, String hadoopVer, String initScript, String cleanupScript,
- boolean useHBaseMetastore, boolean withLlapIo) throws Exception {
+ boolean withLlapIo) throws Exception {
this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
- useHBaseMetastore, withLlapIo, null);
+ withLlapIo, null);
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String confDir, String hadzoopVer, String initScript, String cleanupScript,
- boolean useHBaseMetastore, boolean withLlapIo, FsType fsType)
+ boolean withLlapIo, FsType fsType)
throws Exception {
LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," +
- " hadoopVer={}, initScript={}, cleanupScript={}, useHbaseMetaStore={}, withLlapIo={}," +
+ " hadoopVer={}, initScript={}, cleanupScript={}, withLlapIo={}," +
" fsType={}"
, outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
- useHBaseMetastore, withLlapIo, fsType);
+ withLlapIo, fsType);
Preconditions.checkNotNull(clusterType, "ClusterType cannot be null");
if (fsType != null) {
this.fsType = fsType;
@@ -556,7 +531,6 @@ public class QTestUtil {
}
this.outDir = outDir;
this.logDir = logDir;
- this.useHBaseMetastore = useHBaseMetastore;
this.srcTables=getSrcTables();
this.srcUDFs = getSrcUDFs();
@@ -567,11 +541,7 @@ public class QTestUtil {
}
queryState = new QueryState.Builder().withHiveConf(new HiveConf(Driver.class)).build();
- if (useHBaseMetastore) {
- startMiniHBaseCluster();
- } else {
- conf = queryState.getConf();
- }
+ conf = queryState.getConf();
this.hadoopVer = getHadoopMainVersion(hadoopVer);
qMap = new TreeMap<String, String>();
qSkipSet = new HashSet<String>();
@@ -697,9 +667,6 @@ public class QTestUtil {
sparkSession = null;
}
}
- if (useHBaseMetastore) {
- utility.shutdownMiniCluster();
- }
if (mr != null) {
mr.shutdown();
mr = null;
@@ -2032,8 +1999,7 @@ public class QTestUtil {
for (int i = 0; i < qfiles.length; i++) {
qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20",
initScript == null ? defaultInitScript : initScript,
- cleanupScript == null ? defaultCleanupScript : cleanupScript,
- false, false);
+ cleanupScript == null ? defaultCleanupScript : cleanupScript, false);
qt[i].addFile(qfiles[i]);
qt[i].clearTestSideEffects();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index 31f69a3..30ac6d1 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -54,8 +54,7 @@ public class CoreParseNegative extends CliAdapter{
try {
String hadoopVer = cliConfig.getHadoopVersion();
qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR, null,
- hadoopVer,
- initScript, cleanupScript, false, false);
+ hadoopVer, initScript, cleanupScript, false);
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/pom.xml
----------------------------------------------------------------------
diff --git a/metastore/pom.xml b/metastore/pom.xml
index 073d039..1fa17a7 100644
--- a/metastore/pom.xml
+++ b/metastore/pom.xml
@@ -304,11 +304,6 @@
<mkdir dir="${protobuf.build.dir}"/>
<exec executable="protoc" failonerror="true">
<arg value="--java_out=${protobuf.build.dir}"/>
- <arg value="-I=${protobuf.src.dir}/org/apache/hadoop/hive/metastore/hbase"/>
- <arg value="${protobuf.src.dir}/org/apache/hadoop/hive/metastore/hbase/hbase_metastore_proto.proto"/>
- </exec>
- <exec executable="protoc" failonerror="true">
- <arg value="--java_out=${protobuf.build.dir}"/>
<arg value="-I=${protobuf.src.dir}/org/apache/hadoop/hive/metastore"/>
<arg value="${protobuf.src.dir}/org/apache/hadoop/hive/metastore/metastore.proto"/>
</exec>
[04/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
deleted file mode 100644
index c6a134c..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCache.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHBaseAggregateStatsCache {
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseAggregateStatsCache.class.getName());
-
- @Mock HTableInterface htable;
- private HBaseStore store;
- SortedMap<String, Cell> rows = new TreeMap<>();
-
- @Before
- public void before() throws IOException {
- MockitoAnnotations.initMocks(this);
- HiveConf conf = new HiveConf();
- conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
- store = MockUtils.init(conf, htable, rows);
- store.backdoor().getStatsCache().resetCounters();
- }
-
- private static interface Checker {
- void checkStats(AggrStats aggrStats) throws Exception;
- }
-
- // Do to limitations in the Mock infrastructure we use for HBase testing we can only test
- // this for a single column table and we can't really test hits in hbase, only in memory or
- // build from scratch. But it's still useful to cover many bugs. More in depth testing with
- // multiple columns and with HBase hits is done in TestHBaseAggrStatsCacheIntegration.
-
- @Test
- public void allWithStats() throws Exception {
- String dbName = "default";
- String tableName = "hit";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "boolean", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String>emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(table);
-
- for (List<String> partVals : Arrays.asList(partVals1, partVals2)) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVals.get(0));
- Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String>emptyMap());
- store.addPartition(part);
-
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals.get(0));
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("boolean");
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
- bcsd.setNumFalses(10);
- bcsd.setNumTrues(20);
- bcsd.setNumNulls(30);
- data.setBooleanStats(bcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(cs, partVals);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(2, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("boolean", cso.getColType());
- BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
- Assert.assertEquals(20, bcsd.getNumFalses());
- Assert.assertEquals(40, bcsd.getNumTrues());
- Assert.assertEquals(60, bcsd.getNumNulls());
- }
- };
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- // Check that we had to build it from the stats
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
- // Call again, this time it should come from memory. Also, reverse the name order this time
- // to assure that we still hit.
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
- }
-
-
- @Test
- public void noneWithStats() throws Exception {
- String dbName = "default";
- String tableName = "nws";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "boolean", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String>emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(table);
-
- for (List<String> partVals : Arrays.asList(partVals1, partVals2)) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/nws/ds=" + partVals.get(0));
- Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String>emptyMap());
- store.addPartition(part);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(0, aggrStats.getPartsFound());
- }
- };
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void someNonexistentPartitions() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "boolean", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String>emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(table);
-
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVals1.get(0));
- Partition part = new Partition(partVals1, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String>emptyMap());
- store.addPartition(part);
-
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals1.get(0));
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("double");
- ColumnStatisticsData data = new ColumnStatisticsData();
- DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
- dcsd.setHighValue(1000.2342343);
- dcsd.setLowValue(-20.1234213423);
- dcsd.setNumNulls(30);
- dcsd.setNumDVs(12342);
- data.setDoubleStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(cs, partVals1);
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(1, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("double", cso.getColType());
- DoubleColumnStatsData dcsd = cso.getStatsData().getDoubleStats();
- Assert.assertEquals(1000.23, dcsd.getHighValue(), 0.01);
- Assert.assertEquals(-20.12, dcsd.getLowValue(), 0.01);
- Assert.assertEquals(30, dcsd.getNumNulls());
- Assert.assertEquals(12342, dcsd.getNumDVs());
- }
- };
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- // Check that we had to build it from the stats
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
- // Call again, this time it should come from memory. Also, reverse the name order this time
- // to assure that we still hit.
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
- }
-
- @Test
- public void nonexistentPartitions() throws Exception {
- String dbName = "default";
- String tableName = "nep";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "boolean", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String>emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(table);
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(0, aggrStats.getPartsFound());
- }
- };
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- // Check that we had to build it from the stats
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
- }
- // TODO test invalidation
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
deleted file mode 100644
index 9cf1fb8..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsCacheWithBitVector.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHBaseAggregateStatsCacheWithBitVector {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestHBaseAggregateStatsCacheWithBitVector.class.getName());
-
- @Mock
- HTableInterface htable;
- private HBaseStore store;
- SortedMap<String, Cell> rows = new TreeMap<>();
-
- @Before
- public void before() throws IOException {
- MockitoAnnotations.initMocks(this);
- HiveConf conf = new HiveConf();
- conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
- store = MockUtils.init(conf, htable, rows);
- store.backdoor().getStatsCache().resetCounters();
- }
-
- private static interface Checker {
- void checkStats(AggrStats aggrStats) throws Exception;
- }
-
- @Test
- public void allPartitions() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "boolean", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVals1.get(0));
- Partition part = new Partition(partVals1, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
-
- psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVals2.get(0));
- part = new Partition(partVals2, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
-
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals1.get(0));
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("double");
- ColumnStatisticsData data = new ColumnStatisticsData();
- DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
- dcsd.setHighValue(1000.2342343);
- dcsd.setLowValue(-20.1234213423);
- dcsd.setNumNulls(30);
- dcsd.setNumDVs(12342);
- HyperLogLog hll = HyperLogLog.builder().build();
- hll.addDouble(1);
- hll.addDouble(2);
- hll.addDouble(3);
- dcsd.setBitVectors(hll.serialize());
- data.setDoubleStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVals1);
-
- cs = new ColumnStatistics();
- desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals2.get(0));
- cs.setStatsDesc(desc);
- obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("double");
- data = new ColumnStatisticsData();
- dcsd = new DoubleColumnStatsData();
- dcsd.setHighValue(1000.2342343);
- dcsd.setLowValue(-20.1234213423);
- dcsd.setNumNulls(30);
- dcsd.setNumDVs(12342);
- hll = HyperLogLog.builder().build();
- hll.addDouble(3);
- hll.addDouble(4);
- hll.addDouble(5);
- dcsd.setBitVectors(hll.serialize());
- data.setDoubleStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(cs, partVals2);
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(2, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("double", cso.getColType());
- DoubleColumnStatsData dcsd = cso.getStatsData().getDoubleStats();
- Assert.assertEquals(1000.23, dcsd.getHighValue(), 0.01);
- Assert.assertEquals(-20.12, dcsd.getLowValue(), 0.01);
- Assert.assertEquals(60, dcsd.getNumNulls());
- Assert.assertEquals(5, dcsd.getNumDVs());
- // we do not store the bitvector for the aggrStats.
- // we can store that if it is necessary in the future.
- }
- };
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- // Check that we had to build it from the stats
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
- // Call again, this time it should come from memory. Also, reverse the name
- // order this time
- // to assure that we still hit.
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
deleted file mode 100644
index 4d868b0..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsExtrapolation.java
+++ /dev/null
@@ -1,721 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.StatObjectConverter;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHBaseAggregateStatsExtrapolation {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestHBaseAggregateStatsExtrapolation.class.getName());
-
- @Mock
- HTableInterface htable;
- private HBaseStore store;
- SortedMap<String, Cell> rows = new TreeMap<>();
-
- // NDV will be 3 for the bitVectors
- String bitVectors = null;
- @Before
- public void before() throws IOException {
- MockitoAnnotations.initMocks(this);
- HiveConf conf = new HiveConf();
- conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
- store = MockUtils.init(conf, htable, rows);
- store.backdoor().getStatsCache().resetCounters();
- HyperLogLog hll = HyperLogLog.builder().build();
- hll.addLong(1);
- hll.addLong(2);
- hll.addLong(3);
- bitVectors = hll.serialize();
- }
-
- private static interface Checker {
- void checkStats(AggrStats aggrStats) throws Exception;
- }
-
- @Test
- public void allPartitionsHaveBitVectorStatusLong() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(10, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void allPartitionsHaveBitVectorStatusDecimal() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1_decimal", "decimal", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1_decimal");
- obj.setColType("decimal");
- ColumnStatisticsData data = new ColumnStatisticsData();
- DecimalColumnStatsData dcsd = new DecimalColumnStatsData();
- dcsd.setHighValue(StatObjectConverter.createThriftDecimal("" + (1000 + i)));
- dcsd.setLowValue(StatObjectConverter.createThriftDecimal("" + (-1000 - i)));
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors);
- data.setDecimalStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(10, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1_decimal", cso.getColName());
- Assert.assertEquals("decimal", cso.getColType());
- DecimalColumnStatsData lcsd = cso.getStatsData().getDecimalStats();
- Assert.assertEquals(1009, HBaseUtils.getDoubleValue(lcsd.getHighValue()), 0.01);
- Assert.assertEquals(-1009, HBaseUtils.getDoubleValue(lcsd.getLowValue()), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col1_decimal"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void allPartitionsHaveBitVectorStatusDouble() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1_double", "double", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1_double");
- obj.setColType("double");
- ColumnStatisticsData data = new ColumnStatisticsData();
- DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors);
- data.setDoubleStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(10, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1_double", cso.getColName());
- Assert.assertEquals("double", cso.getColType());
- DoubleColumnStatsData lcsd = cso.getStatsData().getDoubleStats();
- Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col1_double"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void allPartitionsHaveBitVectorStatusString() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1_string", "string", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1_string");
- obj.setColType("string");
- ColumnStatisticsData data = new ColumnStatisticsData();
- StringColumnStatsData dcsd = new StringColumnStatsData();
- dcsd.setAvgColLen(i + 1);
- dcsd.setMaxColLen(i + 10);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors);
- data.setStringStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(10, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1_string", cso.getColName());
- Assert.assertEquals("string", cso.getColType());
- StringColumnStatsData lcsd = cso.getStatsData().getStringStats();
- Assert.assertEquals(10, lcsd.getAvgColLen(), 0.01);
- Assert.assertEquals(19, lcsd.getMaxColLen(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col1_string"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void noPartitionsHaveBitVectorStatus() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col2", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col2");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(i == 0 ? 1 : 10 * i);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(10, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col2", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(90, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col2"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void TwoEndsOfPartitionsHaveBitVectorStatus() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col3", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- if (i < 2 || i > 7) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col3");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i);
- dcsd.setBitVectors(bitVectors);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(4, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col3", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col3"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void MiddleOfPartitionsHaveBitVectorStatus() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col4", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- if (i > 2 && i < 7) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col4");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i);
- dcsd.setBitVectors(bitVectors);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(4, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col4", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1006, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1006, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col4"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col5", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col5");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i);
- dcsd.setBitVectors(bitVectors);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(6, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col5", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(40, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col5"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusDouble() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col5_double", "double", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col5_double");
- obj.setColType("double");
- ColumnStatisticsData data = new ColumnStatisticsData();
- DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i);
- dcsd.setBitVectors(bitVectors);
- data.setDoubleStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(6, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col5_double", cso.getColName());
- Assert.assertEquals("double", cso.getColType());
- DoubleColumnStatsData lcsd = cso.getStatsData().getDoubleStats();
- Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(40, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col5_double"));
- statChecker.checkStats(aggrStats);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
deleted file mode 100644
index 0ad2780..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggregateStatsNDVUniformDist.java
+++ /dev/null
@@ -1,589 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.StatObjectConverter;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHBaseAggregateStatsNDVUniformDist {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestHBaseAggregateStatsNDVUniformDist.class.getName());
-
- @Mock
- HTableInterface htable;
- private HBaseStore store;
- SortedMap<String, Cell> rows = new TreeMap<>();
-
- // NDV will be 3 for bitVectors[0] and 1 for bitVectors[1]
- String bitVectors[] = new String[2];
-
-
- @Before
- public void before() throws IOException {
- MockitoAnnotations.initMocks(this);
- HiveConf conf = new HiveConf();
- conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
- conf.setBoolean(HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION.varname, true);
- store = MockUtils.init(conf, htable, rows);
- store.backdoor().getStatsCache().resetCounters();
- HyperLogLog hll = HyperLogLog.builder().build();
- hll.addLong(1);
- bitVectors[1] = hll.serialize();
- hll = HyperLogLog.builder().build();
- hll.addLong(2);
- hll.addLong(3);
- hll.addLong(3);
- hll.addLong(4);
- bitVectors[0] = hll.serialize();
- }
-
- private static interface Checker {
- void checkStats(AggrStats aggrStats) throws Exception;
- }
-
- @Test
- public void allPartitionsHaveBitVectorStatus() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors[0]);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(10, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void noPartitionsHaveBitVectorStatus() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col2", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col2");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(10, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col2", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1009, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1009, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(91, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col2"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void TwoEndsOfPartitionsHaveBitVectorStatus() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col3", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- if (i < 2 || i > 7) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col3");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors[i / 5]);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(4, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col3", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col3"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void MiddleOfPartitionsHaveBitVectorStatus() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col4", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- if (i > 2 && i < 7) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col4");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors[0]);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(4, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col4", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1006, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1006, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(45, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col4"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusLong() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col5_long", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col5_long");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData dcsd = new LongColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors[i / 5]);
- data.setLongStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(6, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col5_long", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(40, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col5_long"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusDecimal() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col5_decimal", "decimal", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col5_decimal");
- obj.setColType("decimal");
- ColumnStatisticsData data = new ColumnStatisticsData();
- DecimalColumnStatsData dcsd = new DecimalColumnStatsData();
- dcsd.setHighValue(StatObjectConverter.createThriftDecimal("" + (1000 + i)));
- dcsd.setLowValue(StatObjectConverter.createThriftDecimal("" + (-1000 - i)));
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors[i / 5]);
- data.setDecimalStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(6, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col5_decimal", cso.getColName());
- Assert.assertEquals("decimal", cso.getColType());
- DecimalColumnStatsData lcsd = cso.getStatsData().getDecimalStats();
- Assert.assertEquals(1010, HBaseUtils.getDoubleValue(lcsd.getHighValue()), 0.01);
- Assert.assertEquals(-1010, HBaseUtils.getDoubleValue(lcsd.getLowValue()), 0.01);
- Assert.assertEquals(40, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col5_decimal"));
- statChecker.checkStats(aggrStats);
- }
-
- @Test
- public void TwoEndsAndMiddleOfPartitionsHaveBitVectorStatusDouble() throws Exception {
- String dbName = "default";
- String tableName = "snp";
- long now = System.currentTimeMillis();
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col5_double", "double", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String> emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String> emptyMap(), null, null, null);
- store.createTable(table);
-
- List<List<String>> partVals = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- List<String> partVal = Arrays.asList("" + i);
- partVals.add(partVal);
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVal);
- Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String> emptyMap());
- store.addPartition(part);
- if (i == 0 || i == 2 || i == 3 || i == 5 || i == 6 || i == 8) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVal);
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col5_double");
- obj.setColType("double");
- ColumnStatisticsData data = new ColumnStatisticsData();
- DoubleColumnStatsData dcsd = new DoubleColumnStatsData();
- dcsd.setHighValue(1000 + i);
- dcsd.setLowValue(-1000 - i);
- dcsd.setNumNulls(i);
- dcsd.setNumDVs(10 * i + 1);
- dcsd.setBitVectors(bitVectors[i / 5]);
- data.setDoubleStats(dcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
- store.updatePartitionColumnStatistics(cs, partVal);
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(6, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col5_double", cso.getColName());
- Assert.assertEquals("double", cso.getColType());
- DoubleColumnStatsData lcsd = cso.getStatsData().getDoubleStats();
- Assert.assertEquals(1010, lcsd.getHighValue(), 0.01);
- Assert.assertEquals(-1010, lcsd.getLowValue(), 0.01);
- Assert.assertEquals(40, lcsd.getNumNulls());
- Assert.assertEquals(3, lcsd.getNumDVs());
- }
- };
- List<String> partNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- partNames.add("ds=" + i);
- }
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames,
- Arrays.asList("col5_double"));
- statChecker.checkStats(aggrStats);
- }
-}
[06/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionCache.java
deleted file mode 100644
index 08d060f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionCache.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.hive.common.ObjectPair;
-import org.apache.hadoop.hive.metastore.api.Partition;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A cache for partition objects. This is separate from
- * {@link org.apache.hadoop.hive.metastore.hbase.ObjectCache} because we need to access it
- * differently (always by table) and because we need to be able to track whether we are caching
- * all of the partitions for a table or not. Like ObjectCache it is local to a particular thread
- * and thus not synchronized. Also like ObjectCache it is intended to be flushed before each query.
- */
-class PartitionCache {
- // This is a trie. The key to the first map is (dbname, tablename), since partitions are
- // always accessed within the context of the table they belong to. The second map maps
- // partition values (not names) to partitions.
- private Map<ObjectPair<String, String>, TrieValue> cache;
- private final int maxSize;
- private int cacheSize;
- private Counter misses;
- private Counter hits;
- private Counter overflows;
-
- /**
- *
- * @param max maximum number of objects to store in the cache. When max is reached, eviction
- * policy is MRU.
- * @param hits counter to increment when we find an element in the cache
- * @param misses counter to increment when we do not find an element in the cache
- * @param overflows counter to increment when we do not have room for an element in the cache
- */
- PartitionCache(int max, Counter hits, Counter misses, Counter overflows) {
- maxSize = max;
- cache = new HashMap<ObjectPair<String, String>, TrieValue>();
- cacheSize = 0;
- this.hits = hits;
- this.misses = misses;
- this.overflows = overflows;
- }
-
- /**
- * Put a single partition into the cache
- * @param dbName
- * @param tableName
- * @param part
- */
- void put(String dbName, String tableName, Partition part) {
- if (cacheSize < maxSize) {
- ObjectPair<String, String> key = new ObjectPair<String, String>(dbName, tableName);
- TrieValue entry = cache.get(key);
- if (entry == null) {
- entry = new TrieValue(false);
- cache.put(key, entry);
- }
- entry.map.put(part.getValues(), part);
- cacheSize++;
- } else {
- overflows.incr();
- }
- }
-
- /**
- *
- * @param dbName
- * @param tableName
- * @param parts
- * @param allForTable if true indicates that all partitions for this table are present
- */
- void put(String dbName, String tableName, List<Partition> parts, boolean allForTable) {
- if (cacheSize + parts.size() < maxSize) {
- ObjectPair<String, String> key = new ObjectPair<String, String>(dbName, tableName);
- TrieValue entry = cache.get(key);
- if (entry == null) {
- entry = new TrieValue(allForTable);
- cache.put(key, entry);
- }
- for (Partition part : parts) entry.map.put(part.getValues(), part);
- cacheSize += parts.size();
- } else {
- overflows.incr();
- }
- }
-
- /**
- * Will only return a value if all partitions for this table are in the cache. Otherwise you
- * should call {@link #get} individually
- * @param dbName
- * @param tableName
- * @return
- */
- Collection<Partition> getAllForTable(String dbName, String tableName) {
- TrieValue entry = cache.get(new ObjectPair<String, String>(dbName, tableName));
- if (entry != null && entry.hasAllPartitionsForTable) {
- hits.incr();
- return entry.map.values();
- } else {
- misses.incr();
- return null;
- }
- }
-
- Partition get(String dbName, String tableName, List<String> partVals) {
- TrieValue entry = cache.get(new ObjectPair<String, String>(dbName, tableName));
- if (entry != null) {
- hits.incr();
- return entry.map.get(partVals);
- } else {
- misses.incr();
- return null;
- }
- }
-
- void remove(String dbName, String tableName) {
- ObjectPair<String, String> key = new ObjectPair<String, String>(dbName, tableName);
- TrieValue entry = cache.get(key);
- if (entry != null) {
- cacheSize -= entry.map.size();
- cache.remove(key);
- }
- }
-
- void remove(String dbName, String tableName, List<String> partVals) {
- ObjectPair<String, String> key = new ObjectPair<String, String>(dbName, tableName);
- TrieValue entry = cache.get(key);
- if (entry != null && entry.map.remove(partVals) != null) {
- cacheSize--;
- entry.hasAllPartitionsForTable = false;
- }
- }
-
- void flush() {
- cache.clear();
- cacheSize = 0;
- }
-
- static class TrieValue {
- boolean hasAllPartitionsForTable;
- Map<List<String>, Partition> map;
-
- TrieValue(boolean hasAll) {
- hasAllPartitionsForTable = hasAll;
- map = new HashMap<List<String>, Partition>();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionKeyComparator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionKeyComparator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionKeyComparator.java
deleted file mode 100644
index 2b0863d..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/PartitionKeyComparator.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.filter.ByteArrayComparable;
-import org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.PartitionKeyComparator.Operator.Type;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.BytesWritable;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-
-public class PartitionKeyComparator extends ByteArrayComparable {
- private static final Logger LOG = LoggerFactory.getLogger(PartitionKeyComparator.class);
- static class Mark {
- Mark(String value, boolean inclusive) {
- this.value = value;
- this.inclusive = inclusive;
- }
- String value;
- boolean inclusive;
- public String toString() {
- return value + (inclusive?"_":"");
- }
- }
- static class Range {
- Range(String keyName, Mark start, Mark end) {
- this.keyName = keyName;
- this.start = start;
- this.end = end;
- }
- String keyName;
- Mark start;
- Mark end;
- public String toString() {
- return "" + keyName + ":" + (start!=null?start.toString():"") + (end!=null?end.toString():"");
- }
- }
- // Cache the information derived from ranges for performance, including
- // range in native datatype
- static class NativeRange {
- int pos;
- Comparable start;
- Comparable end;
- }
- static class Operator {
- public Operator(Type type, String keyName, String val) {
- this.type = type;
- this.keyName = keyName;
- this.val = val;
- }
- enum Type {
- LIKE, NOTEQUALS
- };
- Type type;
- String keyName;
- String val;
- }
- static class NativeOperator {
- int pos;
- Comparable val;
- }
- String names;
- String types;
- List<Range> ranges;
- List<NativeRange> nativeRanges;
- List<Operator> ops;
- List<NativeOperator> nativeOps;
- Properties serdeProps;
- public PartitionKeyComparator(String names, String types, List<Range> ranges, List<Operator> ops) {
- super(null);
- this.names = names;
- this.types = types;
- this.ranges = ranges;
- this.ops = ops;
- serdeProps = new Properties();
- serdeProps.setProperty(serdeConstants.LIST_COLUMNS, "dbName,tableName," + names);
- serdeProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string," + types);
-
- this.nativeRanges = new ArrayList<NativeRange>(this.ranges.size());
- for (int i=0;i<ranges.size();i++) {
- Range range = ranges.get(i);
- NativeRange nativeRange = new NativeRange();;
- nativeRanges.add(i, nativeRange);
- nativeRange.pos = Arrays.asList(names.split(",")).indexOf(range.keyName);
- TypeInfo expectedType =
- TypeInfoUtils.getTypeInfoFromTypeString(types.split(",")[nativeRange.pos]);
- ObjectInspector outputOI =
- TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
- nativeRange.start = null;
- if (range.start != null) {
- Converter converter = ObjectInspectorConverters.getConverter(
- PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
- nativeRange.start = (Comparable)converter.convert(range.start.value);
- }
- nativeRange.end = null;
- if (range.end != null) {
- Converter converter = ObjectInspectorConverters.getConverter(
- PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
- nativeRange.end = (Comparable)converter.convert(range.end.value);
- }
- }
-
- this.nativeOps = new ArrayList<NativeOperator>(this.ops.size());
- for (int i=0;i<ops.size();i++) {
- Operator op = ops.get(i);
- NativeOperator nativeOp = new NativeOperator();
- nativeOps.add(i, nativeOp);
- nativeOp.pos = ArrayUtils.indexOf(names.split(","), op.keyName);
- TypeInfo expectedType =
- TypeInfoUtils.getTypeInfoFromTypeString(types.split(",")[nativeOp.pos]);
- ObjectInspector outputOI =
- TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
- Converter converter = ObjectInspectorConverters.getConverter(
- PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI);
- nativeOp.val = (Comparable)converter.convert(op.val);
- }
- }
-
- public static PartitionKeyComparator parseFrom(final byte [] bytes) {
- HbaseMetastoreProto.PartitionKeyComparator proto;
- try {
- proto = HbaseMetastoreProto.PartitionKeyComparator.parseFrom(bytes);
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
- List<Range> ranges = new ArrayList<Range>();
- for (HbaseMetastoreProto.PartitionKeyComparator.Range range : proto.getRangeList()) {
- Mark start = null;
- if (range.hasStart()) {
- start = new Mark(range.getStart().getValue(), range.getStart().getInclusive());
- }
- Mark end = null;
- if (range.hasEnd()) {
- end = new Mark(range.getEnd().getValue(), range.getEnd().getInclusive());
- }
- ranges.add(new Range(range.getKey(), start, end));
- }
- List<Operator> ops = new ArrayList<Operator>();
- for (HbaseMetastoreProto.PartitionKeyComparator.Operator op : proto.getOpList()) {
- ops.add(new Operator(Operator.Type.valueOf(op.getType().name()), op.getKey(),
- op.getVal()));
- }
- return new PartitionKeyComparator(proto.getNames(), proto.getTypes(), ranges, ops);
- }
-
- @Override
- public byte[] toByteArray() {
- HbaseMetastoreProto.PartitionKeyComparator.Builder builder =
- HbaseMetastoreProto.PartitionKeyComparator.newBuilder();
- builder.setNames(names);
- builder.setTypes(types);
- for (int i=0;i<ranges.size();i++) {
- Range range = ranges.get(i);
- HbaseMetastoreProto.PartitionKeyComparator.Mark startMark = null;
- if (range.start != null) {
- startMark = HbaseMetastoreProto.PartitionKeyComparator.Mark.newBuilder()
- .setValue(range.start.value)
- .setInclusive(range.start.inclusive)
- .build();
- }
- HbaseMetastoreProto.PartitionKeyComparator.Mark endMark = null;
- if (range.end != null) {
- endMark = HbaseMetastoreProto.PartitionKeyComparator.Mark.newBuilder()
- .setValue(range.end.value)
- .setInclusive(range.end.inclusive)
- .build();
- }
-
- HbaseMetastoreProto.PartitionKeyComparator.Range.Builder rangeBuilder =
- HbaseMetastoreProto.PartitionKeyComparator.Range.newBuilder();
- rangeBuilder.setKey(range.keyName);
- if (startMark != null) {
- rangeBuilder.setStart(startMark);
- }
- if (endMark != null) {
- rangeBuilder.setEnd(endMark);
- }
- builder.addRange(rangeBuilder.build());
- }
- for (int i=0;i<ops.size();i++) {
- Operator op = ops.get(i);
- builder.addOp(HbaseMetastoreProto.PartitionKeyComparator.Operator.newBuilder()
- .setKey(op.keyName)
- .setType(Type.valueOf(op.type.toString()))
- .setVal(op.val).build());
- }
- return builder.build().toByteArray();
- }
-
- @Override
- public int compareTo(byte[] value, int offset, int length) {
- byte[] bytes = Arrays.copyOfRange(value, offset, offset + length);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Get key " + new String(bytes));
- }
- BinarySortableSerDe serDe = new BinarySortableSerDe();
- List deserializedkeys = null;
- try {
- serDe.initialize(new Configuration(), serdeProps);
- deserializedkeys = ((List)serDe.deserialize(new BytesWritable(bytes))).subList(2, 2 + names.split(",").length);
- } catch (SerDeException e) {
- // don't bother with failed deserialization, continue with next key
- return 1;
- }
- for (int i=0;i<ranges.size();i++) {
- Range range = ranges.get(i);
- NativeRange nativeRange = nativeRanges.get(i);
-
- Comparable partVal = (Comparable)deserializedkeys.get(nativeRange.pos);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Try to match range " + partVal + ", start " + nativeRange.start + ", end "
- + nativeRange.end);
- }
- if (range.start == null || range.start.inclusive && partVal.compareTo(nativeRange.start)>=0 ||
- !range.start.inclusive && partVal.compareTo(nativeRange.start)>0) {
- if (range.end == null || range.end.inclusive && partVal.compareTo(nativeRange.end)<=0 ||
- !range.end.inclusive && partVal.compareTo(nativeRange.end)<0) {
- continue;
- }
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Fail to match range " + range.keyName + "-" + partVal + "[" + nativeRange.start
- + "," + nativeRange.end + "]");
- }
- return 1;
- }
-
- for (int i=0;i<ops.size();i++) {
- Operator op = ops.get(i);
- NativeOperator nativeOp = nativeOps.get(i);
- switch (op.type) {
- case LIKE:
- if (!deserializedkeys.get(nativeOp.pos).toString().matches(op.val)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Fail to match operator " + op.keyName + "(" + deserializedkeys.get(nativeOp.pos)
- + ") LIKE " + nativeOp.val);
- }
- return 1;
- }
- break;
- case NOTEQUALS:
- if (nativeOp.val.equals(deserializedkeys.get(nativeOp.pos))) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Fail to match operator " + op.keyName + "(" + deserializedkeys.get(nativeOp.pos)
- + ")!=" + nativeOp.val);
- }
- return 1;
- }
- break;
- }
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("All conditions satisfied:" + deserializedkeys);
- }
- return 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java
deleted file mode 100644
index de3b17b..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/SharedStorageDescriptor.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * A {@link org.apache.hadoop.hive.metastore.api.StorageDescriptor} with most of it's content
- * shallow copied from the underlying storage descriptor. Location and parameters are left alone.
- * To avoid issues when users change the contents, all lists and nested structures (cols, serde,
- * buckets, sortCols, and skewed) are deep copied when they are accessed for reading or writing.
- * (It has to be done on read as well because there's no way to guarantee the user won't change the
- * nested structure or list, which would result in changing every storage descriptor sharing that
- * structure.) Users wishing better performance can call setReadyOnly(), which will prevent the
- * copies.
- */
-public class SharedStorageDescriptor extends StorageDescriptor {
- static final private Logger LOG = LoggerFactory.getLogger(SharedStorageDescriptor.class.getName());
- private boolean colsCopied = false;
- private boolean serdeCopied = false;
- private boolean bucketsCopied = false;
- private boolean sortCopied = false;
- private boolean skewedCopied = false;
-
- SharedStorageDescriptor() {
- }
-
- void setShared(StorageDescriptor shared) {
- if (shared.getCols() != null) super.setCols(shared.getCols());
- // Skip location
- if (shared.getInputFormat() != null) super.setInputFormat(shared.getInputFormat());
- if (shared.getOutputFormat() != null) super.setOutputFormat(shared.getOutputFormat());
- super.setCompressed(shared.isCompressed());
- super.setNumBuckets(shared.getNumBuckets());
- if (shared.getSerdeInfo() != null) super.setSerdeInfo(shared.getSerdeInfo());
- if (shared.getBucketCols() != null) super.setBucketCols(shared.getBucketCols());
- if (shared.getSortCols() != null) super.setSortCols(shared.getSortCols());
- // skip parameters
- if (shared.getSkewedInfo() != null) super.setSkewedInfo(shared.getSkewedInfo());
- super.setStoredAsSubDirectories(shared.isStoredAsSubDirectories());
- }
-
- /**
- * Promise that you'll only use this shared storage descriptor in a read only mode.
- * This prevents the copies of the nested structures and lists when reading them. However, the
- * caller must not change the structures or lists returned to it, as this will change all
- * storage descriptor sharing that list.
- */
- public void setReadOnly() {
- colsCopied = serdeCopied = bucketsCopied = sortCopied = skewedCopied = true;
- }
-
- @Override
- public void addToCols(FieldSchema fs) {
- copyCols();
- super.addToCols(fs);
- }
-
- @Override
- public List<FieldSchema> getCols() {
- copyCols();
- return super.getCols();
- }
-
- @Override
- public void setCols(List<FieldSchema> cols) {
- colsCopied = true;
- super.setCols(cols);
- }
-
- @Override
- public void unsetCols() {
- colsCopied = true;
- super.unsetCols();
- }
-
- @Override
- public Iterator<FieldSchema> getColsIterator() {
- copyCols();
- return super.getColsIterator();
- }
-
- private void copyCols() {
- if (!colsCopied) {
- colsCopied = true;
- if (super.getCols() != null) {
- List<FieldSchema> cols = new ArrayList<FieldSchema>(super.getColsSize());
- for (FieldSchema fs : super.getCols()) cols.add(new FieldSchema(fs));
- super.setCols(cols);
- }
- }
- }
-
- @Override
- public SerDeInfo getSerdeInfo() {
- copySerde();
- return super.getSerdeInfo();
- }
-
- @Override
- public void setSerdeInfo(SerDeInfo serdeInfo) {
- serdeCopied = true;
- super.setSerdeInfo(serdeInfo);
- }
-
- @Override
- public void unsetSerdeInfo() {
- serdeCopied = true;
- super.unsetSerdeInfo();
- }
-
- private void copySerde() {
- if (!serdeCopied) {
- serdeCopied = true;
- if (super.getSerdeInfo() != null) super.setSerdeInfo(new SerDeInfo(super.getSerdeInfo()));
- }
- }
-
- @Override
- public void addToBucketCols(String bucket) {
- copyBucketCols();
- super.addToBucketCols(bucket);
- }
-
- @Override
- public List<String> getBucketCols() {
- copyBucketCols();
- return super.getBucketCols();
- }
-
- @Override
- public void setBucketCols(List<String> buckets) {
- bucketsCopied = true;
- super.setBucketCols(buckets);
- }
-
- @Override
- public void unsetBucketCols() {
- bucketsCopied = true;
- super.unsetBucketCols();
- }
-
- @Override
- public Iterator<String> getBucketColsIterator() {
- copyBucketCols();
- return super.getBucketColsIterator();
- }
-
- private void copyBucketCols() {
- if (!bucketsCopied) {
- bucketsCopied = true;
- if (super.getBucketCols() != null) {
- List<String> buckets = new ArrayList<String>(super.getBucketColsSize());
- for (String bucket : super.getBucketCols()) buckets.add(bucket);
- super.setBucketCols(buckets);
- }
- }
- }
-
- @Override
- public void addToSortCols(Order sort) {
- copySort();
- super.addToSortCols(sort);
- }
-
- @Override
- public List<Order> getSortCols() {
- copySort();
- return super.getSortCols();
- }
-
- @Override
- public void setSortCols(List<Order> sorts) {
- sortCopied = true;
- super.setSortCols(sorts);
- }
-
- @Override
- public void unsetSortCols() {
- sortCopied = true;
- super.unsetSortCols();
- }
-
- @Override
- public Iterator<Order> getSortColsIterator() {
- copySort();
- return super.getSortColsIterator();
- }
-
- private void copySort() {
- if (!sortCopied) {
- sortCopied = true;
- if (super.getSortCols() != null) {
- List<Order> sortCols = new ArrayList<Order>(super.getSortColsSize());
- for (Order sortCol : super.getSortCols()) sortCols.add(new Order(sortCol));
- super.setSortCols(sortCols);
- }
- }
- }
-
- @Override
- public SkewedInfo getSkewedInfo() {
- copySkewed();
- return super.getSkewedInfo();
- }
-
- @Override
- public void setSkewedInfo(SkewedInfo skewedInfo) {
- skewedCopied = true;
- super.setSkewedInfo(skewedInfo);
- }
-
- @Override
- public void unsetSkewedInfo() {
- skewedCopied = true;
- super.unsetSkewedInfo();
- }
-
- private void copySkewed() {
- if (!skewedCopied) {
- skewedCopied = true;
- if (super.getSkewedInfo() != null) super.setSkewedInfo(new SkewedInfo(super.getSkewedInfo()));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
deleted file mode 100644
index 78a962a..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.protobuf.ByteString;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.HiveStatsUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator;
-import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory;
-
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * A cache for stats. This is only intended for use by
- * {@link org.apache.hadoop.hive.metastore.hbase.HBaseReadWrite} and should not be used outside
- * that class.
- */
-class StatsCache {
-
- private static final Logger LOG = LoggerFactory.getLogger(StatsCache.class.getName());
- private static StatsCache self = null;
-
- private LoadingCache<StatsCacheKey, AggrStats> cache;
- private Invalidator invalidator;
- private long runInvalidatorEvery;
- private long maxTimeInCache;
- private boolean invalidatorHasRun;
-
- @VisibleForTesting Counter misses;
- @VisibleForTesting Counter hbaseHits;
- @VisibleForTesting Counter totalGets;
-
- static synchronized StatsCache getInstance(Configuration conf) {
- if (self == null) {
- self = new StatsCache(conf);
- }
- return self;
- }
-
- private StatsCache(final Configuration conf) {
- final StatsCache me = this;
- cache = CacheBuilder.newBuilder()
- .maximumSize(
- HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_CACHE_ENTRIES))
- .expireAfterWrite(HiveConf.getTimeVar(conf,
- HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_MEMORY_TTL, TimeUnit.SECONDS), TimeUnit.SECONDS)
- .build(new CacheLoader<StatsCacheKey, AggrStats>() {
- @Override
- public AggrStats load(StatsCacheKey key) throws Exception {
- boolean useDensityFunctionForNDVEstimation = HiveConf.getBoolVar(conf,
- HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION);
- double ndvTuner = HiveConf.getFloatVar(conf,
- HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_TUNER);
- HBaseReadWrite hrw = HBaseReadWrite.getInstance();
- AggrStats aggrStats = hrw.getAggregatedStats(key.hashed);
- if (aggrStats == null) {
- misses.incr();
- ColumnStatsAggregator aggregator = null;
- aggrStats = new AggrStats();
- LOG.debug("Unable to find aggregated stats for " + key.colName + ", aggregating");
- List<ColumnStatistics> css = hrw.getPartitionStatistics(key.dbName, key.tableName,
- key.partNames, HBaseStore.partNameListToValsList(key.partNames),
- Collections.singletonList(key.colName));
- if (css != null && css.size() > 0) {
- aggrStats.setPartsFound(css.size());
- if (aggregator == null) {
- aggregator = ColumnStatsAggregatorFactory.getColumnStatsAggregator(css.iterator()
- .next().getStatsObj().iterator().next().getStatsData().getSetField(),
- useDensityFunctionForNDVEstimation, ndvTuner);
- }
- ColumnStatisticsObj statsObj = aggregator
- .aggregate(key.colName, key.partNames, css);
- aggrStats.addToColStats(statsObj);
- me.put(key, aggrStats);
- }
- } else {
- hbaseHits.incr();
- }
- return aggrStats;
- }
- });
- misses = new Counter("Stats cache table misses");
- hbaseHits = new Counter("Stats cache table hits");
- totalGets = new Counter("Total get calls to the stats cache");
-
- maxTimeInCache = HiveConf.getTimeVar(conf,
- HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_HBASE_TTL, TimeUnit.SECONDS);
- // We want runEvery in milliseconds, even though we give the default value in the conf in
- // seconds.
- runInvalidatorEvery = HiveConf.getTimeVar(conf,
- HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_INVALIDATOR_FREQUENCY, TimeUnit.MILLISECONDS);
-
- invalidator = new Invalidator();
- invalidator.setDaemon(true);
- invalidator.start();
- }
-
- /**
- * Add an object to the cache.
- * @param key Key for this entry
- * @param aggrStats stats
- * @throws java.io.IOException
- */
- void put(StatsCacheKey key, AggrStats aggrStats) throws IOException {
- HBaseReadWrite.getInstance().putAggregatedStats(key.hashed, key.dbName, key.tableName,
- key.partNames,
- key.colName, aggrStats);
- cache.put(key, aggrStats);
- }
-
- /**
- * Get partition level statistics
- * @param dbName name of database table is in
- * @param tableName name of table
- * @param partNames names of the partitions
- * @param colName of column to get stats for
- * @return stats object for this column, or null if none cached
- * @throws java.io.IOException
- */
-
- AggrStats get(String dbName, String tableName, List<String> partNames, String colName)
- throws IOException {
- totalGets.incr();
- StatsCacheKey key = new StatsCacheKey(dbName, tableName, partNames, colName);
- try {
- return cache.get(key);
- } catch (ExecutionException e) {
- throw new IOException(e);
- }
- }
-
- /**
- * Remove all entries that are related to a particular set of partitions. This should be
- * called when partitions are deleted or stats are updated.
- * @param dbName name of database table is in
- * @param tableName name of table
- * @param partName name of the partition
- * @throws IOException
- */
- void invalidate(String dbName, String tableName, String partName)
- throws IOException {
- invalidator.addToQueue(
- HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry.newBuilder()
- .setDbName(ByteString.copyFrom(dbName.getBytes(HBaseUtils.ENCODING)))
- .setTableName(ByteString.copyFrom(tableName.getBytes(HBaseUtils.ENCODING)))
- .setPartName(ByteString.copyFrom(partName.getBytes(HBaseUtils.ENCODING)))
- .build());
- }
-
- void dumpCounters() {
- LOG.debug(misses.dump());
- LOG.debug(hbaseHits.dump());
- LOG.debug(totalGets.dump());
- }
-
- /**
- * Completely dump the cache from memory, used to test that we can access stats from HBase itself.
- * @throws IOException
- */
- @VisibleForTesting void flushMemory() throws IOException {
- cache.invalidateAll();
- }
-
- @VisibleForTesting void resetCounters() {
- misses.clear();
- hbaseHits.clear();
- totalGets.clear();
- }
-
- @VisibleForTesting void setRunInvalidatorEvery(long runEvery) {
- runInvalidatorEvery = runEvery;
- }
-
- @VisibleForTesting void setMaxTimeInCache(long maxTime) {
- maxTimeInCache = maxTime;
- }
-
- @VisibleForTesting void wakeInvalidator() throws InterruptedException {
- invalidatorHasRun = false;
- // Wait through 2 cycles so we're sure our entry won't be picked as too new.
- Thread.sleep(2 * runInvalidatorEvery);
- invalidator.interrupt();
- while (!invalidatorHasRun) {
- Thread.sleep(10);
- }
- }
-
- static class StatsCacheKey {
- final byte[] hashed;
- String dbName;
- String tableName;
- List<String> partNames;
- String colName;
- private MessageDigest md;
-
- StatsCacheKey(byte[] key) {
- hashed = key;
- }
-
- StatsCacheKey(String dbName, String tableName, List<String> partNames, String colName) {
- this.dbName = dbName;
- this.tableName = tableName;
- this.partNames = partNames;
- this.colName = colName;
-
- try {
- md = MessageDigest.getInstance("MD5");
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException(e);
- }
- md.update(dbName.getBytes(HBaseUtils.ENCODING));
- md.update(tableName.getBytes(HBaseUtils.ENCODING));
- Collections.sort(this.partNames);
- for (String s : partNames) {
- md.update(s.getBytes(HBaseUtils.ENCODING));
- }
- md.update(colName.getBytes(HBaseUtils.ENCODING));
- hashed = md.digest();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null || !(other instanceof StatsCacheKey)) return false;
- StatsCacheKey that = (StatsCacheKey)other;
- return Arrays.equals(hashed, that.hashed);
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(hashed);
- }
- }
-
- private class Invalidator extends Thread {
- private List<HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry> entries = new ArrayList<>();
- private Lock lock = new ReentrantLock();
-
- void addToQueue(HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry entry) {
- lock.lock();
- try {
- entries.add(entry);
- } finally {
- lock.unlock();
- }
- }
-
- @Override
- public void run() {
- while (true) {
- long startedAt = System.currentTimeMillis();
- List<HbaseMetastoreProto.AggrStatsInvalidatorFilter.Entry> thisRun = null;
- lock.lock();
- try {
- if (entries.size() > 0) {
- thisRun = entries;
- entries = new ArrayList<>();
- }
- } finally {
- lock.unlock();
- }
-
- if (thisRun != null) {
- try {
- HbaseMetastoreProto.AggrStatsInvalidatorFilter filter =
- HbaseMetastoreProto.AggrStatsInvalidatorFilter.newBuilder()
- .setRunEvery(runInvalidatorEvery)
- .setMaxCacheEntryLife(maxTimeInCache)
- .addAllToInvalidate(thisRun)
- .build();
- List<StatsCacheKey> keys =
- HBaseReadWrite.getInstance().invalidateAggregatedStats(filter);
- cache.invalidateAll(keys);
- } catch (IOException e) {
- // Not a lot I can do here
- LOG.error("Caught error while invalidating entries in the cache", e);
- }
- }
- invalidatorHasRun = true;
-
- try {
- sleep(runInvalidatorEvery - (System.currentTimeMillis() - startedAt));
- } catch (InterruptedException e) {
- LOG.warn("Interupted while sleeping", e);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/TephraHBaseConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/TephraHBaseConnection.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/TephraHBaseConnection.java
deleted file mode 100644
index f66200f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/TephraHBaseConnection.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import co.cask.tephra.TransactionAware;
-import co.cask.tephra.TransactionContext;
-import co.cask.tephra.TransactionFailureException;
-import co.cask.tephra.TransactionManager;
-import co.cask.tephra.TransactionSystemClient;
-import co.cask.tephra.distributed.ThreadLocalClientProvider;
-import co.cask.tephra.distributed.TransactionServiceClient;
-import co.cask.tephra.hbase10.TransactionAwareHTable;
-import co.cask.tephra.hbase10.coprocessor.TransactionProcessor;
-import co.cask.tephra.inmemory.InMemoryTxSystemClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.twill.discovery.InMemoryDiscoveryService;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A class that uses Tephra for transaction management.
- */
-public class TephraHBaseConnection extends VanillaHBaseConnection {
- static final private Logger LOG = LoggerFactory.getLogger(TephraHBaseConnection.class.getName());
-
- private Map<String, TransactionAware> txnTables;
- private TransactionContext txn;
- private TransactionSystemClient txnClient;
-
- TephraHBaseConnection() {
- super();
- txnTables = new HashMap<String, TransactionAware>();
- }
-
- @Override
- public void connect() throws IOException {
- super.connect();
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEST)) {
- LOG.debug("Using an in memory client transaction system for testing");
- TransactionManager txnMgr = new TransactionManager(conf);
- txnMgr.startAndWait();
- txnClient = new InMemoryTxSystemClient(txnMgr);
- } else {
- // TODO should enable use of ZKDiscoveryService if users want it
- LOG.debug("Using real client transaction system for production");
- txnClient = new TransactionServiceClient(conf,
- new ThreadLocalClientProvider(conf, new InMemoryDiscoveryService()));
- }
- for (String tableName : HBaseReadWrite.tableNames) {
- txnTables.put(tableName, new TransactionAwareHTable(super.getHBaseTable(tableName, true)));
- }
- txn = new TransactionContext(txnClient, txnTables.values());
- }
-
- @Override
- public void beginTransaction() throws IOException {
- try {
- txn.start();
- LOG.debug("Started txn in tephra");
- } catch (TransactionFailureException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void commitTransaction() throws IOException {
- try {
- txn.finish();
- LOG.debug("Finished txn in tephra");
- } catch (TransactionFailureException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void rollbackTransaction() throws IOException {
- try {
- txn.abort();
- LOG.debug("Aborted txn in tephra");
- } catch (TransactionFailureException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void flush(HTableInterface htab) throws IOException {
- // NO-OP as we want to flush at commit time
- }
-
- @Override
- protected HTableDescriptor buildDescriptor(String tableName, List<byte[]> columnFamilies)
- throws IOException {
- HTableDescriptor tableDesc = super.buildDescriptor(tableName, columnFamilies);
- tableDesc.addCoprocessor(TransactionProcessor.class.getName());
- return tableDesc;
- }
-
- @Override
- public HTableInterface getHBaseTable(String tableName, boolean force) throws IOException {
- // Ignore force, it will mess up our previous creation of the tables.
- return (TransactionAwareHTable)txnTables.get(tableName);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/VanillaHBaseConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/VanillaHBaseConnection.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/VanillaHBaseConnection.java
deleted file mode 100644
index e631580..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/VanillaHBaseConnection.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A pass through to a simple HBase connection. This has no transactions.
- */
-public class VanillaHBaseConnection implements HBaseConnection {
- static final private Logger LOG = LoggerFactory.getLogger(VanillaHBaseConnection.class.getName());
-
- protected HConnection conn;
- protected Map<String, HTableInterface> tables;
- protected Configuration conf;
-
- VanillaHBaseConnection() {
- tables = new HashMap<String, HTableInterface>();
- }
-
- @Override
- public void connect() throws IOException {
- if (conf == null) throw new RuntimeException("Must call getConf before connect");
- conn = HConnectionManager.createConnection(conf);
- }
-
- @Override
- public void close() throws IOException {
- for (HTableInterface htab : tables.values()) htab.close();
- }
-
- @Override
- public void beginTransaction() throws IOException {
-
- }
-
- @Override
- public void commitTransaction() throws IOException {
-
- }
-
- @Override
- public void rollbackTransaction() throws IOException {
-
- }
-
- @Override
- public void flush(HTableInterface htab) throws IOException {
- htab.flushCommits();
- }
-
- @Override
- public void createHBaseTable(String tableName, List<byte[]> columnFamilies)
- throws IOException {
- HBaseAdmin admin = new HBaseAdmin(conn);
- LOG.info("Creating HBase table " + tableName);
- admin.createTable(buildDescriptor(tableName, columnFamilies));
- admin.close();
- }
-
- protected HTableDescriptor buildDescriptor(String tableName, List<byte[]> columnFamilies)
- throws IOException {
- HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
- for (byte[] cf : columnFamilies) {
- tableDesc.addFamily(new HColumnDescriptor(cf));
- }
- return tableDesc;
- }
-
- @Override
- public HTableInterface getHBaseTable(String tableName) throws IOException {
- return getHBaseTable(tableName, false);
- }
-
- @Override
- public HTableInterface getHBaseTable(String tableName, boolean force) throws IOException {
- HTableInterface htab = tables.get(tableName);
- if (htab == null) {
- LOG.debug("Trying to connect to table " + tableName);
- try {
- htab = conn.getTable(tableName);
- // Calling gettable doesn't actually connect to the region server, it's very light
- // weight, so call something else so we actually reach out and touch the region server
- // and see if the table is there.
- if (force) htab.get(new Get("nosuchkey".getBytes(HBaseUtils.ENCODING)));
- } catch (IOException e) {
- LOG.info("Caught exception when table was missing");
- return null;
- }
- htab.setAutoFlushTo(false);
- tables.put(tableName, htab);
- }
- return htab;
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
deleted file mode 100644
index e6c836b..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BinaryColumnStatsAggregator.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-public class BinaryColumnStatsAggregator extends ColumnStatsAggregator {
-
- @Override
- public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
- List<ColumnStatistics> css) throws MetaException {
- ColumnStatisticsObj statsObj = null;
- BinaryColumnStatsData aggregateData = null;
- String colType = null;
- for (ColumnStatistics cs : css) {
- if (cs.getStatsObjSize() != 1) {
- throw new MetaException(
- "The number of columns should be exactly one in aggrStats, but found "
- + cs.getStatsObjSize());
- }
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- if (statsObj == null) {
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
- .getStatsData().getSetField());
- }
- BinaryColumnStatsData newData = cso.getStatsData().getBinaryStats();
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- }
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- columnStatisticsData.setBinaryStats(aggregateData);
- statsObj.setStatsData(columnStatisticsData);
- return statsObj;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
deleted file mode 100644
index a34bc9f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/BooleanColumnStatsAggregator.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-public class BooleanColumnStatsAggregator extends ColumnStatsAggregator {
-
- @Override
- public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
- List<ColumnStatistics> css) throws MetaException {
- ColumnStatisticsObj statsObj = null;
- BooleanColumnStatsData aggregateData = null;
- String colType = null;
- for (ColumnStatistics cs : css) {
- if (cs.getStatsObjSize() != 1) {
- throw new MetaException(
- "The number of columns should be exactly one in aggrStats, but found "
- + cs.getStatsObjSize());
- }
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- if (statsObj == null) {
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
- .getStatsData().getSetField());
- }
- BooleanColumnStatsData newData = cso.getStatsData().getBooleanStats();
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
- aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- }
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- columnStatisticsData.setBooleanStats(aggregateData);
- statsObj.setStatsData(columnStatisticsData);
- return statsObj;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
deleted file mode 100644
index a52e5e5..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregator.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-public abstract class ColumnStatsAggregator {
- public boolean useDensityFunctionForNDVEstimation;
- public double ndvTuner;
- public abstract ColumnStatisticsObj aggregate(String colName, List<String> partNames,
- List<ColumnStatistics> css) throws MetaException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
deleted file mode 100644
index 173e06f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-
-public class ColumnStatsAggregatorFactory {
-
- private ColumnStatsAggregatorFactory() {
- }
-
- public static ColumnStatsAggregator getColumnStatsAggregator(_Fields type,
- boolean useDensityFunctionForNDVEstimation, double ndvTuner) {
- ColumnStatsAggregator agg;
- switch (type) {
- case BOOLEAN_STATS:
- agg = new BooleanColumnStatsAggregator();
- break;
- case LONG_STATS:
- agg = new LongColumnStatsAggregator();
- break;
- case DATE_STATS:
- agg = new DateColumnStatsAggregator();
- break;
- case DOUBLE_STATS:
- agg = new DoubleColumnStatsAggregator();
- break;
- case STRING_STATS:
- agg = new StringColumnStatsAggregator();
- break;
- case BINARY_STATS:
- agg = new BinaryColumnStatsAggregator();
- break;
- case DECIMAL_STATS:
- agg = new DecimalColumnStatsAggregator();
- break;
- default:
- throw new RuntimeException("Woh, bad. Unknown stats type " + type.toString());
- }
- agg.useDensityFunctionForNDVEstimation = useDensityFunctionForNDVEstimation;
- agg.ndvTuner = ndvTuner;
- return agg;
- }
-
- public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
- ColumnStatisticsObj cso = new ColumnStatisticsObj();
- ColumnStatisticsData csd = new ColumnStatisticsData();
- cso.setColName(colName);
- cso.setColType(colType);
- switch (type) {
- case BOOLEAN_STATS:
- csd.setBooleanStats(new BooleanColumnStatsData());
- break;
-
- case LONG_STATS:
- csd.setLongStats(new LongColumnStatsData());
- break;
-
- case DATE_STATS:
- csd.setDateStats(new DateColumnStatsData());
- break;
-
- case DOUBLE_STATS:
- csd.setDoubleStats(new DoubleColumnStatsData());
- break;
-
- case STRING_STATS:
- csd.setStringStats(new StringColumnStatsData());
- break;
-
- case BINARY_STATS:
- csd.setBinaryStats(new BinaryColumnStatsData());
- break;
-
- case DECIMAL_STATS:
- csd.setDecimalStats(new DecimalColumnStatsData());
- break;
-
- default:
- throw new RuntimeException("Woh, bad. Unknown stats type!");
- }
-
- cso.setStatsData(csd);
- return cso;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
deleted file mode 100644
index 5924c3e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/DecimalColumnStatsAggregator.java
+++ /dev/null
@@ -1,371 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.StatObjectConverter;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.hbase.HBaseUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implements
- IExtrapolatePartStatus {
-
- private static final Logger LOG = LoggerFactory.getLogger(DecimalColumnStatsAggregator.class);
-
- @Override
- public ColumnStatisticsObj aggregate(String colName, List<String> partNames,
- List<ColumnStatistics> css) throws MetaException {
- ColumnStatisticsObj statsObj = null;
-
- // check if all the ColumnStatisticsObjs contain stats and all the ndv are
- // bitvectors
- boolean doAllPartitionContainStats = partNames.size() == css.size();
- LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats);
- NumDistinctValueEstimator ndvEstimator = null;
- String colType = null;
- for (ColumnStatistics cs : css) {
- if (cs.getStatsObjSize() != 1) {
- throw new MetaException(
- "The number of columns should be exactly one in aggrStats, but found "
- + cs.getStatsObjSize());
- }
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- if (statsObj == null) {
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso
- .getStatsData().getSetField());
- }
- if (!cso.getStatsData().getDecimalStats().isSetBitVectors()
- || cso.getStatsData().getDecimalStats().getBitVectors().length() == 0) {
- ndvEstimator = null;
- break;
- } else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(cso.getStatsData().getDecimalStats().getBitVectors());
- if (ndvEstimator == null) {
- ndvEstimator = estimator;
- } else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
- break;
- }
- }
- }
- }
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- if (doAllPartitionContainStats || css.size() < 2) {
- DecimalColumnStatsData aggregateData = null;
- long lowerBound = 0;
- long higherBound = 0;
- double densityAvgSum = 0.0;
- for (ColumnStatistics cs : css) {
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
- lowerBound = Math.max(lowerBound, newData.getNumDVs());
- higherBound += newData.getNumDVs();
- densityAvgSum += (HBaseUtils.getDoubleValue(newData.getHighValue()) - HBaseUtils
- .getDoubleValue(newData.getLowValue())) / newData.getNumDVs();
- if (ndvEstimator != null) {
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- if (HBaseUtils.getDoubleValue(aggregateData.getLowValue()) < HBaseUtils
- .getDoubleValue(newData.getLowValue())) {
- aggregateData.setLowValue(aggregateData.getLowValue());
- } else {
- aggregateData.setLowValue(newData.getLowValue());
- }
- if (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) > HBaseUtils
- .getDoubleValue(newData.getHighValue())) {
- aggregateData.setHighValue(aggregateData.getHighValue());
- } else {
- aggregateData.setHighValue(newData.getHighValue());
- }
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- }
- }
- if (ndvEstimator != null) {
- // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
- // use uniform distribution assumption because we can merge bitvectors
- // to get a good estimation.
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- } else {
- long estimation;
- if (useDensityFunctionForNDVEstimation) {
- // We have estimation, lowerbound and higherbound. We use estimation
- // if it is between lowerbound and higherbound.
- double densityAvg = densityAvgSum / partNames.size();
- estimation = (long) ((HBaseUtils.getDoubleValue(aggregateData.getHighValue()) - HBaseUtils
- .getDoubleValue(aggregateData.getLowValue())) / densityAvg);
- if (estimation < lowerBound) {
- estimation = lowerBound;
- } else if (estimation > higherBound) {
- estimation = higherBound;
- }
- } else {
- estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
- }
- aggregateData.setNumDVs(estimation);
- }
- columnStatisticsData.setDecimalStats(aggregateData);
- } else {
- // we need extrapolation
- LOG.debug("start extrapolation for " + colName);
- Map<String, Integer> indexMap = new HashMap<String, Integer>();
- for (int index = 0; index < partNames.size(); index++) {
- indexMap.put(partNames.get(index), index);
- }
- Map<String, Double> adjustedIndexMap = new HashMap<String, Double>();
- Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<String, ColumnStatisticsData>();
- // while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
- double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
- // if not every partition uses bitvector for ndv, we just fall back to
- // the traditional extrapolation methods.
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (HBaseUtils.getDoubleValue(newData.getHighValue()) - HBaseUtils
- .getDoubleValue(newData.getLowValue())) / newData.getNumDVs();
- }
- adjustedIndexMap.put(partName, (double) indexMap.get(partName));
- adjustedStatsMap.put(partName, cso.getStatsData());
- }
- } else {
- // we first merge all the adjacent bitvectors that we could merge and
- // derive new partition names and index.
- StringBuilder pseudoPartName = new StringBuilder();
- double pseudoIndexSum = 0;
- int length = 0;
- int curIndex = -1;
- DecimalColumnStatsData aggregateData = null;
- for (ColumnStatistics cs : css) {
- String partName = cs.getStatsDesc().getPartName();
- ColumnStatisticsObj cso = cs.getStatsObjIterator().next();
- DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats();
- // newData.isSetBitVectors() should be true for sure because we
- // already checked it before.
- if (indexMap.get(partName) != curIndex) {
- // There is bitvector, but it is not adjacent to the previous ones.
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setDecimalStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) - HBaseUtils
- .getDoubleValue(aggregateData.getLowValue())) / aggregateData.getNumDVs();
- }
- // reset everything
- pseudoPartName = new StringBuilder();
- pseudoIndexSum = 0;
- length = 0;
- ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- aggregateData = null;
- }
- curIndex = indexMap.get(partName);
- pseudoPartName.append(partName);
- pseudoIndexSum += curIndex;
- length++;
- curIndex++;
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- if (HBaseUtils.getDoubleValue(aggregateData.getLowValue()) < HBaseUtils
- .getDoubleValue(newData.getLowValue())) {
- aggregateData.setLowValue(aggregateData.getLowValue());
- } else {
- aggregateData.setLowValue(newData.getLowValue());
- }
- if (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) > HBaseUtils
- .getDoubleValue(newData.getHighValue())) {
- aggregateData.setHighValue(aggregateData.getHighValue());
- } else {
- aggregateData.setHighValue(newData.getHighValue());
- }
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- ndvEstimator.mergeEstimators(NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(newData.getBitVectors()));
- }
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setDecimalStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += (HBaseUtils.getDoubleValue(aggregateData.getHighValue()) - HBaseUtils
- .getDoubleValue(aggregateData.getLowValue())) / aggregateData.getNumDVs();
- }
- }
- }
- extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap,
- adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
- }
- statsObj.setStatsData(columnStatisticsData);
- LOG.debug("Ndv estimatation for " + colName + " is "
- + columnStatisticsData.getDecimalStats().getNumDVs());
- return statsObj;
- }
-
- @Override
- public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
- int numPartsWithStats, Map<String, Double> adjustedIndexMap,
- Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- DecimalColumnStatsData extrapolateDecimalData = new DecimalColumnStatsData();
- Map<String, DecimalColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
- for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
- extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDecimalStats());
- }
- List<Map.Entry<String, DecimalColumnStatsData>> list = new LinkedList<Map.Entry<String, DecimalColumnStatsData>>(
- extractedAdjustedStatsMap.entrySet());
- // get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
- public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
- Map.Entry<String, DecimalColumnStatsData> o2) {
- return o1.getValue().getLowValue().compareTo(o2.getValue().getLowValue());
- }
- });
- double minInd = adjustedIndexMap.get(list.get(0).getKey());
- double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double lowValue = 0;
- double min = HBaseUtils.getDoubleValue(list.get(0).getValue().getLowValue());
- double max = HBaseUtils.getDoubleValue(list.get(list.size() - 1).getValue().getLowValue());
- if (minInd == maxInd) {
- lowValue = min;
- } else if (minInd < maxInd) {
- // left border is the min
- lowValue = (max - (max - min) * maxInd / (maxInd - minInd));
- } else {
- // right border is the min
- lowValue = (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
- }
-
- // get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
- public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
- Map.Entry<String, DecimalColumnStatsData> o2) {
- return o1.getValue().getHighValue().compareTo(o2.getValue().getHighValue());
- }
- });
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- double highValue = 0;
- min = HBaseUtils.getDoubleValue(list.get(0).getValue().getHighValue());
- max = HBaseUtils.getDoubleValue(list.get(list.size() - 1).getValue().getHighValue());
- if (minInd == maxInd) {
- highValue = min;
- } else if (minInd < maxInd) {
- // right border is the max
- highValue = (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- highValue = (min + (max - min) * minInd / (minInd - maxInd));
- }
-
- // get the #nulls
- long numNulls = 0;
- for (Map.Entry<String, DecimalColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
- numNulls += entry.getValue().getNumNulls();
- }
- // we scale up sumNulls based on the number of partitions
- numNulls = numNulls * numParts / numPartsWithStats;
-
- // get the ndv
- long ndv = 0;
- long ndvMin = 0;
- long ndvMax = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, DecimalColumnStatsData>>() {
- public int compare(Map.Entry<String, DecimalColumnStatsData> o1,
- Map.Entry<String, DecimalColumnStatsData> o2) {
- return o1.getValue().getNumDVs() < o2.getValue().getNumDVs() ? -1 : 1;
- }
- });
- long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
- long higherBound = 0;
- for (Map.Entry<String, DecimalColumnStatsData> entry : list) {
- higherBound += entry.getValue().getNumDVs();
- }
- if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
- ndv = (long) ((highValue - lowValue) / densityAvg);
- if (ndv < lowerBound) {
- ndv = lowerBound;
- } else if (ndv > higherBound) {
- ndv = higherBound;
- }
- } else {
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- ndvMin = list.get(0).getValue().getNumDVs();
- ndvMax = list.get(list.size() - 1).getValue().getNumDVs();
- if (minInd == maxInd) {
- ndv = ndvMin;
- } else if (minInd < maxInd) {
- // right border is the max
- ndv = (long) (ndvMin + (ndvMax - ndvMin) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- ndv = (long) (ndvMin + (ndvMax - ndvMin) * minInd / (minInd - maxInd));
- }
- }
- extrapolateDecimalData.setLowValue(StatObjectConverter.createThriftDecimal(String
- .valueOf(lowValue)));
- extrapolateDecimalData.setHighValue(StatObjectConverter.createThriftDecimal(String
- .valueOf(highValue)));
- extrapolateDecimalData.setNumNulls(numNulls);
- extrapolateDecimalData.setNumDVs(ndv);
- extrapolateData.setDecimalStats(extrapolateDecimalData);
- }
-}
[09/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
deleted file mode 100644
index b4f8734..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.conf.Configuration;
-
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * A tool to dump contents from the HBase store in a human readable form
- */
-public class HBaseSchemaTool {
-
- public static void main(String[] args) {
- Options options = new Options();
-
- options.addOption(OptionBuilder
- .withLongOpt("help")
- .withDescription("You're looking at it")
- .create('h'));
-
- options.addOption(OptionBuilder
- .withLongOpt("install")
- .withDescription("Install the schema onto an HBase cluster.")
- .create('i'));
-
- options.addOption(OptionBuilder
- .withLongOpt("key")
- .withDescription("Key to scan with. This should be an exact key (not a regular expression")
- .hasArg()
- .create('k'));
-
- options.addOption(OptionBuilder
- .withLongOpt("list-tables")
- .withDescription("List tables in HBase metastore")
- .create('l'));
-
- options.addOption(OptionBuilder
- .withLongOpt("regex-key")
- .withDescription("Regular expression to scan keys with.")
- .hasArg()
- .create('r'));
-
- options.addOption(OptionBuilder
- .withLongOpt("table")
- .withDescription("HBase metastore table to scan")
- .hasArg()
- .create('t'));
-
- CommandLine cli = null;
- try {
- cli = new GnuParser().parse(options, args);
- } catch (ParseException e) {
- System.err.println("Parse Exception: " + e.getMessage());
- usage(options);
- return;
- }
-
- if (cli.hasOption('h')) {
- usage(options);
- return;
- }
-
- Configuration conf = new Configuration();
-
- if (cli.hasOption('i')) {
- new HBaseSchemaTool().install(conf, System.err);
- return;
- }
-
- String key = null;
- if (cli.hasOption('k')) key = cli.getOptionValue('k');
- String regex = null;
- if (cli.hasOption('r')) regex = cli.getOptionValue('r');
- if (key != null && regex != null) {
- usage(options);
- return;
- }
- if (key == null && regex == null) regex = ".*";
-
- // I do this in the object rather than in the static main so that it's easier to test.
- new HBaseSchemaTool().go(cli.hasOption('l'), cli.getOptionValue('t'), key, regex, conf,
- System.out, System.err);
- }
-
- private static void usage(Options options) {
- HelpFormatter formatter = new HelpFormatter();
- String header = "This tool dumps contents of your hbase metastore. You need to specify\n" +
- "the table to dump. You can optionally specify a regular expression on the key for\n" +
- "the table. Keep in mind that the key is often a compound. For partitions regular\n" +
- "expressions are not used because non-string values are\nstored in binary. Instead for " +
- "partition you can specify as much of the exact prefix as you want. So you can give " +
- "dbname.tablename or dbname.tablename.pval1...";
- String footer = "If neither key or regex is provided a regex of .* will be assumed. You\n" +
- "cannot set both key and regex.";
- formatter.printHelp("hbaseschematool", header, options, footer);
- return;
- }
-
- @VisibleForTesting void go(boolean listTables, String table, String key, String regex,
- Configuration conf, PrintStream out, PrintStream err) {
- List<String> lines = new ArrayList<>();
- if (listTables) {
- lines = Arrays.asList(HBaseReadWrite.tableNames);
- } else {
- // If they've used '.' as a key separator we need to replace it with the separator used by
- // HBaseUtils
- if (key != null) key = key.replace('.', HBaseUtils.KEY_SEPARATOR);
- try {
- HBaseReadWrite.setConf(conf);
- HBaseReadWrite hrw = HBaseReadWrite.getInstance();
- if (table.equalsIgnoreCase(HBaseReadWrite.DB_TABLE)) {
- if (key != null) lines.add(hrw.printDatabase(key));
- else lines.addAll(hrw.printDatabases(regex));
- } else if (table.equalsIgnoreCase(HBaseReadWrite.FUNC_TABLE)) {
- if (key != null) lines.add(hrw.printFunction(key));
- else lines.addAll(hrw.printFunctions(regex));
- } else if (table.equalsIgnoreCase(HBaseReadWrite.GLOBAL_PRIVS_TABLE)) {
- // Ignore whatever they passed, there's always only either one or zero global privileges
- lines.add(hrw.printGlobalPrivs());
- } else if (table.equalsIgnoreCase(HBaseReadWrite.PART_TABLE)) {
- if (key != null) lines.add(hrw.printPartition(key));
- else lines.addAll(hrw.printPartitions(regex));
- } else if (table.equalsIgnoreCase(HBaseReadWrite.USER_TO_ROLE_TABLE)) {
- if (key != null) lines.add(hrw.printRolesForUser(key));
- else lines.addAll(hrw.printRolesForUsers(regex));
- } else if (table.equalsIgnoreCase(HBaseReadWrite.ROLE_TABLE)) {
- if (key != null) lines.add(hrw.printRole(key));
- else lines.addAll(hrw.printRoles(regex));
- } else if (table.equalsIgnoreCase(HBaseReadWrite.TABLE_TABLE)) {
- if (key != null) lines.add(hrw.printTable(key));
- else lines.addAll(hrw.printTables(regex));
- } else if (table.equalsIgnoreCase(HBaseReadWrite.SD_TABLE)) {
- if (key != null) lines.add(hrw.printStorageDescriptor(Base64.decodeBase64(key)));
- else lines.addAll(hrw.printStorageDescriptors());
- } else if (table.equalsIgnoreCase(HBaseReadWrite.SECURITY_TABLE)) {
- // We always print all of security, we don't worry about finding particular entries.
- lines.addAll(hrw.printSecurity());
- } else if (table.equalsIgnoreCase(HBaseReadWrite.SEQUENCES_TABLE)) {
- // We always print all of sequences, we don't worry about finding particular entries.
- lines.addAll(hrw.printSequences());
- } else {
- err.println("Unknown table: " + table);
- return;
- }
- } catch (Exception e) {
- err.println("Caught exception " + e.getClass() + " with message: " + e.getMessage());
- return;
- }
- }
- for (String line : lines) out.println(line);
- }
-
- @VisibleForTesting void install(Configuration conf, PrintStream err) {
- try {
- // We need to set the conf because createTablesIfNotExist will get a thread local version
- // which requires that the configuration object be set.
- HBaseReadWrite.setConf(conf);
- HBaseReadWrite.createTablesIfNotExist();
- } catch (Exception e) {
- err.println("Caught exception " + e.getClass() + " with message: " + e.getMessage());
- return;
- }
- }
-}
[17/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
HIVE-17234 Remove HBase metastore from master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5e061557
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5e061557
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5e061557
Branch: refs/heads/master
Commit: 5e0615573c1fecf3ece7ff238b76bbce45a0337c
Parents: c9c7162
Author: Alan Gates <ga...@hortonworks.com>
Authored: Fri Aug 4 14:08:56 2017 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Fri Aug 4 14:08:56 2017 -0700
----------------------------------------------------------------------
bin/ext/hbaseimport.sh | 27 -
bin/ext/hbaseschematool.sh | 27 -
.../org/apache/hadoop/hive/conf/HiveConf.java | 52 -
.../metastore/hbase/HBaseIntegrationTests.java | 112 -
.../TestHBaseAggrStatsCacheIntegration.java | 691 -
.../hive/metastore/hbase/TestHBaseImport.java | 775 -
.../hbase/TestHBaseMetastoreMetrics.java | 130 -
.../metastore/hbase/TestHBaseMetastoreSql.java | 223 -
.../metastore/hbase/TestHBaseSchemaTool.java | 582 -
.../metastore/hbase/TestHBaseSchemaTool2.java | 59 -
.../hbase/TestHBaseStoreIntegration.java | 1801 -
.../hbase/TestStorageDescriptorSharing.java | 191 -
.../hadoop/hive/ql/TestLocationQueries.java | 2 +-
.../hadoop/hive/accumulo/AccumuloQTestUtil.java | 2 +-
.../hive/cli/control/AbstractCliConfig.java | 4 +-
.../control/AbstractCoreBlobstoreCliDriver.java | 3 +-
.../hadoop/hive/cli/control/CoreCliDriver.java | 4 +-
.../hive/cli/control/CoreCompareCliDriver.java | 2 +-
.../hive/cli/control/CoreNegativeCliDriver.java | 2 +-
.../hive/cli/control/CorePerfCliDriver.java | 2 +-
.../hadoop/hive/hbase/HBaseQTestUtil.java | 2 +-
.../metastore/hbase/HBaseStoreTestUtil.java | 45 -
.../org/apache/hadoop/hive/ql/QTestUtil.java | 54 +-
.../hadoop/hive/ql/parse/CoreParseNegative.java | 3 +-
metastore/pom.xml | 5 -
.../metastore/hbase/HbaseMetastoreProto.java | 46709 -----------------
.../hadoop/hive/metastore/MetaStoreUtils.java | 97 +
.../hive/metastore/cache/SharedCache.java | 6 +-
.../aggr/BinaryColumnStatsAggregator.java | 64 +
.../aggr/BooleanColumnStatsAggregator.java | 65 +
.../columnstats/aggr/ColumnStatsAggregator.java | 33 +
.../aggr/ColumnStatsAggregatorFactory.java | 113 +
.../aggr/DecimalColumnStatsAggregator.java | 371 +
.../aggr/DoubleColumnStatsAggregator.java | 345 +
.../aggr/IExtrapolatePartStatus.java | 47 +
.../aggr/LongColumnStatsAggregator.java | 344 +
.../merge/BinaryColumnStatsMerger.java | 35 +
.../merge/BooleanColumnStatsMerger.java | 35 +
.../columnstats/merge/ColumnStatsMerger.java | 31 +
.../merge/ColumnStatsMergerFactory.java | 123 +
.../merge/DateColumnStatsMerger.java | 61 +
.../merge/DecimalColumnStatsMerger.java | 63 +
.../merge/DoubleColumnStatsMerger.java | 56 +
.../merge/LongColumnStatsMerger.java | 56 +
.../merge/StringColumnStatsMerger.java | 56 +
.../hbase/AggrStatsInvalidatorFilter.java | 120 -
.../hadoop/hive/metastore/hbase/Counter.java | 53 -
.../hive/metastore/hbase/HBaseConnection.java | 96 -
.../metastore/hbase/HBaseFilterPlanUtil.java | 612 -
.../hive/metastore/hbase/HBaseImport.java | 619 -
.../hive/metastore/hbase/HBaseReadWrite.java | 2942 --
.../hive/metastore/hbase/HBaseSchemaTool.java | 192 -
.../hadoop/hive/metastore/hbase/HBaseStore.java | 2969 --
.../hadoop/hive/metastore/hbase/HBaseUtils.java | 1781 -
.../hive/metastore/hbase/ObjectCache.java | 81 -
.../hive/metastore/hbase/PartitionCache.java | 168 -
.../metastore/hbase/PartitionKeyComparator.java | 292 -
.../hbase/SharedStorageDescriptor.java | 251 -
.../hadoop/hive/metastore/hbase/StatsCache.java | 325 -
.../metastore/hbase/TephraHBaseConnection.java | 127 -
.../metastore/hbase/VanillaHBaseConnection.java | 137 -
.../stats/BinaryColumnStatsAggregator.java | 64 -
.../stats/BooleanColumnStatsAggregator.java | 65 -
.../hbase/stats/ColumnStatsAggregator.java | 33 -
.../stats/ColumnStatsAggregatorFactory.java | 113 -
.../stats/DecimalColumnStatsAggregator.java | 371 -
.../stats/DoubleColumnStatsAggregator.java | 345 -
.../hbase/stats/IExtrapolatePartStatus.java | 47 -
.../hbase/stats/LongColumnStatsAggregator.java | 344 -
.../stats/merge/BinaryColumnStatsMerger.java | 35 -
.../stats/merge/BooleanColumnStatsMerger.java | 35 -
.../hbase/stats/merge/ColumnStatsMerger.java | 31 -
.../stats/merge/ColumnStatsMergerFactory.java | 123 -
.../stats/merge/DateColumnStatsMerger.java | 61 -
.../stats/merge/DecimalColumnStatsMerger.java | 63 -
.../stats/merge/DoubleColumnStatsMerger.java | 56 -
.../stats/merge/LongColumnStatsMerger.java | 56 -
.../stats/merge/StringColumnStatsMerger.java | 56 -
.../metastore/hbase/hbase_metastore_proto.proto | 368 -
.../hadoop/hive/metastore/hbase/MockUtils.java | 219 -
.../hbase/TestHBaseAggregateStatsCache.java | 316 -
...stHBaseAggregateStatsCacheWithBitVector.java | 193 -
.../TestHBaseAggregateStatsExtrapolation.java | 721 -
.../TestHBaseAggregateStatsNDVUniformDist.java | 589 -
.../hbase/TestHBaseFilterPlanUtil.java | 483 -
.../hive/metastore/hbase/TestHBaseStore.java | 1978 -
.../hbase/TestHBaseStoreBitVector.java | 622 -
.../metastore/hbase/TestHBaseStoreCached.java | 365 -
.../hbase/TestSharedStorageDescriptor.java | 152 -
89 files changed, 2018 insertions(+), 70161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/bin/ext/hbaseimport.sh
----------------------------------------------------------------------
diff --git a/bin/ext/hbaseimport.sh b/bin/ext/hbaseimport.sh
deleted file mode 100644
index 638cdcf..0000000
--- a/bin/ext/hbaseimport.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-THISSERVICE=hbaseimport
-export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
-
-hbaseimport () {
- CLASS=org.apache.hadoop.hive.metastore.hbase.HBaseImport
- HIVE_OPTS=''
- execHiveCmd $CLASS "$@"
-}
-
-hbaseimport_help () {
- echo "usage ./hive hbaseimport"
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/bin/ext/hbaseschematool.sh
----------------------------------------------------------------------
diff --git a/bin/ext/hbaseschematool.sh b/bin/ext/hbaseschematool.sh
deleted file mode 100644
index 4d4570a..0000000
--- a/bin/ext/hbaseschematool.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-THISSERVICE=hbaseschematool
-export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
-
-hbaseschematool () {
- CLASS=org.apache.hadoop.hive.metastore.hbase.HBaseSchemaTool
- HIVE_OPTS=''
- execHiveCmd $CLASS "$@"
-}
-
-hbaseschematool_help () {
- echo "usage ./hive hbaseschematool [-d <dbname>] <cmd>"
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 14235b5..62a65c9 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -291,21 +291,6 @@ public class HiveConf extends Configuration {
HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_FULL,
HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL,
HiveConf.ConfVars.METASTORE_FASTPATH,
- HiveConf.ConfVars.METASTORE_HBASE_CATALOG_CACHE_SIZE,
- HiveConf.ConfVars.METASTORE_HBASE_AGGREGATE_STATS_CACHE_SIZE,
- HiveConf.ConfVars.METASTORE_HBASE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS,
- HiveConf.ConfVars.METASTORE_HBASE_AGGREGATE_STATS_CACHE_FALSE_POSITIVE_PROBABILITY,
- HiveConf.ConfVars.METASTORE_HBASE_AGGREGATE_STATS_CACHE_MAX_VARIANCE,
- HiveConf.ConfVars.METASTORE_HBASE_CACHE_TIME_TO_LIVE,
- HiveConf.ConfVars.METASTORE_HBASE_CACHE_MAX_WRITER_WAIT,
- HiveConf.ConfVars.METASTORE_HBASE_CACHE_MAX_READER_WAIT,
- HiveConf.ConfVars.METASTORE_HBASE_CACHE_MAX_FULL,
- HiveConf.ConfVars.METASTORE_HBASE_CACHE_CLEAN_UNTIL,
- HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS,
- HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_CACHE_ENTRIES,
- HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_MEMORY_TTL,
- HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_INVALIDATOR_FREQUENCY,
- HiveConf.ConfVars.METASTORE_HBASE_AGGR_STATS_HBASE_TTL,
HiveConf.ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS
};
@@ -622,43 +607,6 @@ public class HiveConf extends Configuration {
"undefined and most likely undesired behavior will result"),
METASTORE_FS_HANDLER_THREADS_COUNT("hive.metastore.fshandler.threads", 15,
"Number of threads to be allocated for metastore handler for fs operations."),
- METASTORE_HBASE_CATALOG_CACHE_SIZE("hive.metastore.hbase.catalog.cache.size", 50000, "Maximum number of " +
- "objects we will place in the hbase metastore catalog cache. The objects will be divided up by " +
- "types that we need to cache."),
- METASTORE_HBASE_AGGREGATE_STATS_CACHE_SIZE("hive.metastore.hbase.aggregate.stats.cache.size", 10000,
- "Maximum number of aggregate stats nodes that we will place in the hbase metastore aggregate stats cache."),
- METASTORE_HBASE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS("hive.metastore.hbase.aggregate.stats.max.partitions", 10000,
- "Maximum number of partitions that are aggregated per cache node."),
- METASTORE_HBASE_AGGREGATE_STATS_CACHE_FALSE_POSITIVE_PROBABILITY("hive.metastore.hbase.aggregate.stats.false.positive.probability",
- (float) 0.01, "Maximum false positive probability for the Bloom Filter used in each aggregate stats cache node (default 1%)."),
- METASTORE_HBASE_AGGREGATE_STATS_CACHE_MAX_VARIANCE("hive.metastore.hbase.aggregate.stats.max.variance", (float) 0.1,
- "Maximum tolerable variance in number of partitions between a cached node and our request (default 10%)."),
- METASTORE_HBASE_CACHE_TIME_TO_LIVE("hive.metastore.hbase.cache.ttl", "600s", new TimeValidator(TimeUnit.SECONDS),
- "Number of seconds for a cached node to be active in the cache before they become stale."),
- METASTORE_HBASE_CACHE_MAX_WRITER_WAIT("hive.metastore.hbase.cache.max.writer.wait", "5000ms", new TimeValidator(TimeUnit.MILLISECONDS),
- "Number of milliseconds a writer will wait to acquire the writelock before giving up."),
- METASTORE_HBASE_CACHE_MAX_READER_WAIT("hive.metastore.hbase.cache.max.reader.wait", "1000ms", new TimeValidator(TimeUnit.MILLISECONDS),
- "Number of milliseconds a reader will wait to acquire the readlock before giving up."),
- METASTORE_HBASE_CACHE_MAX_FULL("hive.metastore.hbase.cache.max.full", (float) 0.9,
- "Maximum cache full % after which the cache cleaner thread kicks in."),
- METASTORE_HBASE_CACHE_CLEAN_UNTIL("hive.metastore.hbase.cache.clean.until", (float) 0.8,
- "The cleaner thread cleans until cache reaches this % full size."),
- METASTORE_HBASE_CONNECTION_CLASS("hive.metastore.hbase.connection.class",
- "org.apache.hadoop.hive.metastore.hbase.VanillaHBaseConnection",
- "Class used to connection to HBase"),
- METASTORE_HBASE_AGGR_STATS_CACHE_ENTRIES("hive.metastore.hbase.aggr.stats.cache.entries",
- 10000, "How many in stats objects to cache in memory"),
- METASTORE_HBASE_AGGR_STATS_MEMORY_TTL("hive.metastore.hbase.aggr.stats.memory.ttl", "60s",
- new TimeValidator(TimeUnit.SECONDS),
- "Number of seconds stats objects live in memory after they are read from HBase."),
- METASTORE_HBASE_AGGR_STATS_INVALIDATOR_FREQUENCY(
- "hive.metastore.hbase.aggr.stats.invalidator.frequency", "5s",
- new TimeValidator(TimeUnit.SECONDS),
- "How often the stats cache scans its HBase entries and looks for expired entries"),
- METASTORE_HBASE_AGGR_STATS_HBASE_TTL("hive.metastore.hbase.aggr.stats.hbase.ttl", "604800s",
- new TimeValidator(TimeUnit.SECONDS),
- "Number of seconds stats entries live in HBase cache after they are created. They may be" +
- " invalided by updates or partition drops before this. Default is one week."),
METASTORE_HBASE_FILE_METADATA_THREADS("hive.metastore.hbase.file.metadata.threads", 1,
"Number of threads to use to read file metadata in background to cache it."),
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
deleted file mode 100644
index d4cd818..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Integration tests with HBase Mini-cluster for HBaseStore
- */
-public class HBaseIntegrationTests {
-
- private static final Logger LOG = LoggerFactory.getLogger(HBaseIntegrationTests.class.getName());
-
- protected static HBaseTestingUtility utility;
- protected static HBaseAdmin admin;
- protected static Map<String, String> emptyParameters = new HashMap<>();
- protected static HiveConf conf;
-
- protected HBaseStore store;
- protected Driver driver;
-
- protected static void startMiniCluster() throws Exception {
- String connectionClassName =
- System.getProperty(HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS.varname);
- boolean testingTephra =
- connectionClassName != null && connectionClassName.equals(TephraHBaseConnection.class.getName());
- if (testingTephra) {
- LOG.info("Testing with Tephra");
- }
- Configuration hbaseConf = HBaseConfiguration.create();
- hbaseConf.setInt("hbase.master.info.port", -1);
- utility = new HBaseTestingUtility(hbaseConf);
- utility.startMiniCluster();
- conf = new HiveConf(utility.getConfiguration(), HBaseIntegrationTests.class);
- admin = utility.getHBaseAdmin();
- HBaseStoreTestUtil.initHBaseMetastore(admin, null);
- }
-
- protected static void shutdownMiniCluster() throws Exception {
- utility.shutdownMiniCluster();
- }
-
- protected void setupConnection() throws IOException {
-
- }
-
- protected void setupDriver() {
- // This chicanery is necessary to make the driver work. Hive tests need the pfile file
- // system, while the hbase one uses something else. So first make sure we've configured our
- // hbase connection, then get a new config file and populate it as desired.
- HBaseReadWrite.setConf(conf);
- conf = new HiveConf();
- conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
- conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
- "org.apache.hadoop.hive.metastore.hbase.HBaseStore");
- conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true);
- conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- // Setup so we can test SQL standard auth
- conf.setBoolVar(HiveConf.ConfVars.HIVE_TEST_AUTHORIZATION_SQLSTD_HS2_MODE, true);
- conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
- SQLStdHiveAuthorizerFactoryForTest.class.getName());
- conf.setVar(HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
- SessionStateConfigUserAuthenticator.class.getName());
- conf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED, true);
- conf.setVar(HiveConf.ConfVars.USERS_IN_ADMIN_ROLE, System.getProperty("user.name"));
- conf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE,"nonstrict");
- //HBaseReadWrite.setTestConnection(hconn);
-
- SessionState.start(new CliSessionState(conf));
- driver = new Driver(conf);
- }
-
- protected void setupHBaseStore() {
- // Turn off caching, as we want to test actual interaction with HBase
- conf.setBoolean(HBaseReadWrite.NO_CACHE_CONF, true);
- store = new HBaseStore();
- store.setConf(conf);
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggrStatsCacheIntegration.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggrStatsCacheIntegration.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggrStatsCacheIntegration.java
deleted file mode 100644
index 51d96dd..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseAggrStatsCacheIntegration.java
+++ /dev/null
@@ -1,691 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Integration tests with HBase Mini-cluster for HBaseStore
- */
-public class TestHBaseAggrStatsCacheIntegration extends HBaseIntegrationTests {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseStoreIntegration.class.getName());
-
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- @BeforeClass
- public static void startup() throws Exception {
- HBaseIntegrationTests.startMiniCluster();
- }
-
- @AfterClass
- public static void shutdown() throws Exception {
- HBaseIntegrationTests.shutdownMiniCluster();
- }
-
- @Before
- public void setup() throws IOException {
- setupConnection();
- setupHBaseStore();
- store.backdoor().getStatsCache().resetCounters();
- }
-
- private static interface Checker {
- void checkStats(AggrStats aggrStats) throws Exception;
- }
-
- @Test
- public void hit() throws Exception {
- String dbName = "default";
- String tableName = "hit";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "boolean", "nocomment"));
- cols.add(new FieldSchema("col2", "varchar", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String>emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(table);
-
- for (List<String> partVals : Arrays.asList(partVals1, partVals2)) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/hit/ds=" + partVals.get(0));
- Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String>emptyMap());
- store.addPartition(part);
-
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals.get(0));
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("boolean");
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
- bcsd.setNumFalses(10);
- bcsd.setNumTrues(20);
- bcsd.setNumNulls(30);
- data.setBooleanStats(bcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- obj = new ColumnStatisticsObj();
- obj.setColName("col2");
- obj.setColType("varchar");
- data = new ColumnStatisticsData();
- StringColumnStatsData scsd = new StringColumnStatsData();
- scsd.setAvgColLen(10.3);
- scsd.setMaxColLen(2000);
- scsd.setNumNulls(3);
- scsd.setNumDVs(12342);
- data.setStringStats(scsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(cs, partVals);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(2, aggrStats.getPartsFound());
- Assert.assertEquals(2, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("boolean", cso.getColType());
- BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
- Assert.assertEquals(20, bcsd.getNumFalses());
- Assert.assertEquals(40, bcsd.getNumTrues());
- Assert.assertEquals(60, bcsd.getNumNulls());
-
- cso = aggrStats.getColStats().get(1);
- Assert.assertEquals("col2", cso.getColName());
- Assert.assertEquals("varchar", cso.getColType());
- StringColumnStatsData scsd = cso.getStatsData().getStringStats();
- Assert.assertEquals(10.3, scsd.getAvgColLen(), 0.1);
- Assert.assertEquals(2000, scsd.getMaxColLen());
- Assert.assertEquals(6, scsd.getNumNulls());
- Assert.assertEquals(12342, scsd.getNumDVs());
- }
- };
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1", "col2"));
- statChecker.checkStats(aggrStats);
-
- // Check that we had to build it from the stats
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
- // Call again, this time it should come from memory. Also, reverse the name order this time
- // to assure that we still hit.
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1", "col2"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(4, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
- store.backdoor().getStatsCache().flushMemory();
- // Call again, this time it should come from hbase
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1", "col2"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(2, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(6, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
- }
-
- @Test
- public void someWithStats() throws Exception {
- String dbName = "default";
- String tableName = "psws";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "long", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String>emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int)now, (int)now, 0, sd, partCols,
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(table);
-
- boolean first = true;
- for (List<String> partVals : Arrays.asList(partVals1, partVals2)) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/psws/ds=" + partVals.get(0));
- Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String>emptyMap());
- store.addPartition(part);
-
- if (first) {
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals.get(0));
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("long");
- ColumnStatisticsData data = new ColumnStatisticsData();
- LongColumnStatsData lcsd = new LongColumnStatsData();
- lcsd.setHighValue(192L);
- lcsd.setLowValue(-20L);
- lcsd.setNumNulls(30);
- lcsd.setNumDVs(32);
- data.setLongStats(lcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(cs, partVals);
- first = false;
- }
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(1, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("long", cso.getColType());
- LongColumnStatsData lcsd = cso.getStatsData().getLongStats();
- Assert.assertEquals(192L, lcsd.getHighValue());
- Assert.assertEquals(-20L, lcsd.getLowValue());
- Assert.assertEquals(30, lcsd.getNumNulls());
- Assert.assertEquals(32, lcsd.getNumDVs());
- }
- };
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- // Check that we had to build it from the stats
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
- // Call again, this time it should come from memory. Also, reverse the name order this time
- // to assure that we still hit.
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
- store.backdoor().getStatsCache().flushMemory();
- // Call again, this time it should come from hbase
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(1, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(3, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
- }
-
- @Test
- public void invalidation() throws Exception {
- try {
- String dbName = "default";
- String tableName = "invalidation";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- List<String> partVals3 = Arrays.asList("tomorrow");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "boolean", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String>emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(table);
-
- for (List<String> partVals : Arrays.asList(partVals1, partVals2, partVals3)) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
- Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String>emptyMap());
- store.addPartition(part);
-
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals.get(0));
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("boolean");
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
- bcsd.setNumFalses(10);
- bcsd.setNumTrues(20);
- bcsd.setNumNulls(30);
- data.setBooleanStats(bcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(cs, partVals);
- }
-
- Checker statChecker = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(2, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("boolean", cso.getColType());
- BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
- Assert.assertEquals(20, bcsd.getNumFalses());
- Assert.assertEquals(40, bcsd.getNumTrues());
- Assert.assertEquals(60, bcsd.getNumNulls());
- }
- };
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- // Check that we had to build it from the stats
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
- // Call again, this time it should come from memory. Also, reverse the name order this time
- // to assure that we still hit.
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(1, store.backdoor().getStatsCache().misses.getCnt());
-
- // Now call a different combination to get it in memory too
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(3, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(4, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
- // wake the invalidator and check again to make sure it isn't too aggressive about
- // removing our stuff.
- store.backdoor().getStatsCache().wakeInvalidator();
-
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(5, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
- // Update statistics for 'tomorrow'
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals3.get(0));
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("boolean");
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
- bcsd.setNumFalses(100);
- bcsd.setNumTrues(200);
- bcsd.setNumNulls(300);
- data.setBooleanStats(bcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- Checker afterUpdate = new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(2, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("boolean", cso.getColType());
- BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
- Assert.assertEquals(110, bcsd.getNumFalses());
- Assert.assertEquals(220, bcsd.getNumTrues());
- Assert.assertEquals(330, bcsd.getNumNulls());
- }
- };
-
- store.updatePartitionColumnStatistics(cs, partVals3);
-
- store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
- store.backdoor().getStatsCache().wakeInvalidator();
-
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
- afterUpdate.checkStats(aggrStats);
-
- // Check that we missed, which means this aggregate was dropped from the cache.
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(6, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
-
- // Check that our other aggregate is still in the cache.
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
- statChecker.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(7, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
-
- // Drop 'yesterday', so our first aggregate should be dumped from memory and hbase
- store.dropPartition(dbName, tableName, partVals2);
-
- store.backdoor().getStatsCache().wakeInvalidator();
-
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
- new Checker() {
- @Override
- public void checkStats(AggrStats aggrStats) throws Exception {
- Assert.assertEquals(1, aggrStats.getPartsFound());
- Assert.assertEquals(1, aggrStats.getColStatsSize());
- ColumnStatisticsObj cso = aggrStats.getColStats().get(0);
- Assert.assertEquals("col1", cso.getColName());
- Assert.assertEquals("boolean", cso.getColType());
- BooleanColumnStatsData bcsd = cso.getStatsData().getBooleanStats();
- Assert.assertEquals(10, bcsd.getNumFalses());
- Assert.assertEquals(20, bcsd.getNumTrues());
- Assert.assertEquals(30, bcsd.getNumNulls());
- }
- }.checkStats(aggrStats);
-
- // Check that we missed, which means this aggregate was dropped from the cache.
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(8, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(4, store.backdoor().getStatsCache().misses.getCnt());
-
- // Check that our other aggregate is still in the cache.
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
- afterUpdate.checkStats(aggrStats);
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(9, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(4, store.backdoor().getStatsCache().misses.getCnt());
- } finally {
- store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
- store.backdoor().getStatsCache().setMaxTimeInCache(500000);
- store.backdoor().getStatsCache().wakeInvalidator();
- }
- }
-
- @Test
- public void alterInvalidation() throws Exception {
- try {
- String dbName = "default";
- String tableName = "ai";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- List<String> partVals3 = Arrays.asList("tomorrow");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "boolean", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String>emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(table);
-
- Partition[] partitions = new Partition[3];
- int partnum = 0;
- for (List<String> partVals : Arrays.asList(partVals1, partVals2, partVals3)) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
- Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String>emptyMap());
- partitions[partnum++] = part;
- store.addPartition(part);
-
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals.get(0));
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("boolean");
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
- bcsd.setNumFalses(10);
- bcsd.setNumTrues(20);
- bcsd.setNumNulls(30);
- data.setBooleanStats(bcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(cs, partVals);
- }
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=tomorrow"), Arrays.asList("col1"));
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-
- // Check that we had to build it from the stats
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
- // wake the invalidator and check again to make sure it isn't too aggressive about
- // removing our stuff.
- store.backdoor().getStatsCache().wakeInvalidator();
-
- Partition newPart = new Partition(partitions[2]);
- newPart.setLastAccessTime((int)System.currentTimeMillis());
- store.alterPartition(dbName, tableName, partVals3, newPart);
-
- store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
- store.backdoor().getStatsCache().wakeInvalidator();
-
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
-
- // Check that we missed, which means this aggregate was dropped from the cache.
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(3, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
-
- // Check that our other aggregate is still in the cache.
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(4, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
- } finally {
- store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
- store.backdoor().getStatsCache().setMaxTimeInCache(500000);
- store.backdoor().getStatsCache().wakeInvalidator();
- }
- }
-
- @Test
- public void altersInvalidation() throws Exception {
- try {
- String dbName = "default";
- String tableName = "asi";
- List<String> partVals1 = Arrays.asList("today");
- List<String> partVals2 = Arrays.asList("yesterday");
- List<String> partVals3 = Arrays.asList("tomorrow");
- long now = System.currentTimeMillis();
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "boolean", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, Collections.<String, String>emptyMap());
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("ds", "string", ""));
- Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
- Collections.<String, String>emptyMap(), null, null, null);
- store.createTable(table);
-
- Partition[] partitions = new Partition[3];
- int partnum = 0;
- for (List<String> partVals : Arrays.asList(partVals1, partVals2, partVals3)) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/default/invalidation/ds=" + partVals.get(0));
- Partition part = new Partition(partVals, dbName, tableName, (int) now, (int) now, psd,
- Collections.<String, String>emptyMap());
- partitions[partnum++] = part;
- store.addPartition(part);
-
- ColumnStatistics cs = new ColumnStatistics();
- ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName);
- desc.setLastAnalyzed(now);
- desc.setPartName("ds=" + partVals.get(0));
- cs.setStatsDesc(desc);
- ColumnStatisticsObj obj = new ColumnStatisticsObj();
- obj.setColName("col1");
- obj.setColType("boolean");
- ColumnStatisticsData data = new ColumnStatisticsData();
- BooleanColumnStatsData bcsd = new BooleanColumnStatsData();
- bcsd.setNumFalses(10);
- bcsd.setNumTrues(20);
- bcsd.setNumNulls(30);
- data.setBooleanStats(bcsd);
- obj.setStatsData(data);
- cs.addToStatsObj(obj);
-
- store.updatePartitionColumnStatistics(cs, partVals);
- }
-
- AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=tomorrow"), Arrays.asList("col1"));
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=today", "ds=yesterday"), Arrays.asList("col1"));
-
- // Check that we had to build it from the stats
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(2, store.backdoor().getStatsCache().misses.getCnt());
-
- // wake the invalidator and check again to make sure it isn't too aggressive about
- // removing our stuff.
- store.backdoor().getStatsCache().wakeInvalidator();
-
- Partition[] newParts = new Partition[2];
- newParts[0] = new Partition(partitions[0]);
- newParts[0].setLastAccessTime((int)System.currentTimeMillis());
- newParts[1] = new Partition(partitions[2]);
- newParts[1].setLastAccessTime((int) System.currentTimeMillis());
- store.alterPartitions(dbName, tableName, Arrays.asList(partVals1, partVals3),
- Arrays.asList(newParts));
-
- store.backdoor().getStatsCache().setRunInvalidatorEvery(100);
- store.backdoor().getStatsCache().wakeInvalidator();
-
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=tomorrow", "ds=today"), Arrays.asList("col1"));
-
- // Check that we missed, which means this aggregate was dropped from the cache.
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(3, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(3, store.backdoor().getStatsCache().misses.getCnt());
-
- // Check that our other aggregate got dropped too
- aggrStats = store.get_aggr_stats_for(dbName, tableName,
- Arrays.asList("ds=yesterday", "ds=today"), Arrays.asList("col1"));
-
- Assert.assertEquals(0, store.backdoor().getStatsCache().hbaseHits.getCnt());
- Assert.assertEquals(4, store.backdoor().getStatsCache().totalGets.getCnt());
- Assert.assertEquals(4, store.backdoor().getStatsCache().misses.getCnt());
- } finally {
- store.backdoor().getStatsCache().setRunInvalidatorEvery(5000);
- store.backdoor().getStatsCache().setMaxTimeInCache(500000);
- store.backdoor().getStatsCache().wakeInvalidator();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java
deleted file mode 100644
index b1d3174..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java
+++ /dev/null
@@ -1,775 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.TestObjectStore;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.FunctionType;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.ResourceType;
-import org.apache.hadoop.hive.metastore.api.ResourceUri;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Test that import from an RDBMS based metastore works
- */
-public class TestHBaseImport extends HBaseIntegrationTests {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseImport.class.getName());
-
- private static final String[] tableNames = new String[] {"allnonparttable", "allparttable"};
- private static final String[] partVals = new String[] {"na", "emea", "latam", "apac"};
- private static final String[] funcNames = new String[] {"allfunc1", "allfunc2"};
- private static final String[] indexNames = new String[] {"allindex1", "allindex2"};
- private static final String[] pkNames = new String[] {"allnonparttable_pk", "allparttable_pk"};
- private static final String[] fkNames = new String[] {"", "allparttable_fk"};
-
- private static final List<Integer> masterKeySeqs = new ArrayList<Integer>();
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @BeforeClass
- public static void startup() throws Exception {
- HBaseIntegrationTests.startMiniCluster();
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
- TestObjectStore.dropAllStoreObjects(rdbms);
- }
-
- @AfterClass
- public static void shutdown() throws Exception {
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
- TestObjectStore.dropAllStoreObjects(rdbms);
- for (int seq : masterKeySeqs) {
- rdbms.removeMasterKey(seq);
- }
- HBaseIntegrationTests.shutdownMiniCluster();
- }
-
- @Before
- public void setup() throws IOException {
- setupConnection();
- setupHBaseStore();
- }
-
- @Test
- public void importAll() throws Exception {
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"alldb1", "alldb2"};
- String[] roles = new String[] {"allrole1", "allrole2"};
- String[] tokenIds = new String[] {"alltokenid1", "alltokenid2"};
- String[] tokens = new String[] {"alltoken1", "alltoken2"};
- String[] masterKeys = new String[] {"allmk1", "allmk2"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
- int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
- int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
-
- HBaseImport importer = new HBaseImport("-a");
- importer.setConnections(rdbms, store);
- importer.run();
-
- for (int i = 0; i < roles.length; i++) {
- Role role = store.getRole(roles[i]);
- Assert.assertNotNull(role);
- Assert.assertEquals(roles[i], role.getRoleName());
- }
- // Make sure there aren't any extra roles
- Assert.assertEquals(baseNumRoles + 2, store.listRoleNames().size());
-
- for (int i = 0; i < dbNames.length; i++) {
- Database db = store.getDatabase(dbNames[i]);
- Assert.assertNotNull(db);
- // check one random value in the db rather than every value
- Assert.assertEquals("file:/tmp", db.getLocationUri());
-
- Table table = store.getTable(db.getName(), tableNames[0]);
- Assert.assertNotNull(table);
- Assert.assertEquals(now, table.getLastAccessTime());
- Assert.assertEquals("input", table.getSd().getInputFormat());
-
- table = store.getTable(db.getName(), tableNames[1]);
- Assert.assertNotNull(table);
-
- for (int j = 0; j < partVals.length; j++) {
- Partition part = store.getPartition(dbNames[i], tableNames[1], Arrays.asList(partVals[j]));
- Assert.assertNotNull(part);
- Assert.assertEquals("file:/tmp/region=" + partVals[j], part.getSd().getLocation());
- }
-
- Assert.assertEquals(4, store.getPartitions(dbNames[i], tableNames[1], -1).size());
- // Including two index table
- Assert.assertEquals(4, store.getAllTables(dbNames[i]).size());
-
- Assert.assertEquals(2, store.getIndexes(dbNames[i], tableNames[0], -1).size());
- Assert.assertEquals(0, store.getIndexes(dbNames[i], tableNames[1], -1).size());
-
- Assert.assertEquals(2, store.getFunctions(dbNames[i], "*").size());
- for (int j = 0; j < funcNames.length; j++) {
- Assert.assertNotNull(store.getFunction(dbNames[i], funcNames[j]));
- }
- }
-
- Assert.assertEquals(baseNumDbs + 2, store.getAllDatabases().size());
-
- // I can't test total number of tokens or master keys because the import grabs all and copies
- // them, which means it grabs the ones imported by importSecurity test (if it's already run).
- // Depending on it already running would make the tests order dependent, which junit doesn't
- // guarantee.
- for (int i = 0; i < tokenIds.length; i++) {
- Assert.assertEquals(tokens[i], store.getToken(tokenIds[i]));
- }
- String[] hbaseKeys = store.getMasterKeys();
- Set<String> keys = new HashSet<>(Arrays.asList(hbaseKeys));
- for (int i = 0; i < masterKeys.length; i++) {
- Assert.assertTrue(keys.contains(masterKeys[i]));
- }
- }
-
- @Test
- public void importOneDb() throws Exception {
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"onedbdb1", "onedbdb2"};
- String[] roles = new String[] {"onedbrole1", "onedbrole2"};
- String[] tokenIds = new String[] {"onedbtokenid1", "onedbtokenid2"};
- String[] tokens = new String[] {"onedbtoken1", "onedbtoken2"};
- String[] masterKeys = new String[] {"onedbmk1", "onedbmk2"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
- int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
- int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
- int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
- store.getAllTokenIdentifiers().size();
- int baseNumKeys = store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
- HBaseImport importer = new HBaseImport("-d", dbNames[0]);
- importer.setConnections(rdbms, store);
- importer.run();
-
- // Make sure there aren't any extra roles
- Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
- Database db = store.getDatabase(dbNames[0]);
- Assert.assertNotNull(db);
- // check one random value in the db rather than every value
- Assert.assertEquals("file:/tmp", db.getLocationUri());
-
- Table table = store.getTable(db.getName(), tableNames[0]);
- Assert.assertNotNull(table);
- Assert.assertEquals(now, table.getLastAccessTime());
- Assert.assertEquals("input", table.getSd().getInputFormat());
-
- table = store.getTable(db.getName(), tableNames[1]);
- Assert.assertNotNull(table);
-
- for (int j = 0; j < partVals.length; j++) {
- Partition part = store.getPartition(dbNames[0], tableNames[1], Arrays.asList(partVals[j]));
- Assert.assertNotNull(part);
- Assert.assertEquals("file:/tmp/region=" + partVals[j], part.getSd().getLocation());
- }
-
- Assert.assertEquals(4, store.getPartitions(dbNames[0], tableNames[1], -1).size());
- // Including two index table
- Assert.assertEquals(4, store.getAllTables(dbNames[0]).size());
-
- Assert.assertEquals(2, store.getIndexes(dbNames[0], tableNames[0], -1).size());
- Assert.assertEquals(0, store.getIndexes(dbNames[0], tableNames[1], -1).size());
-
- Assert.assertEquals(2, store.getFunctions(dbNames[0], "*").size());
- for (int j = 0; j < funcNames.length; j++) {
- Assert.assertNotNull(store.getFunction(dbNames[0], funcNames[j]));
- }
-
- Assert.assertEquals(baseNumDbs + 1, store.getAllDatabases().size());
-
- Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
- String[] hbaseKeys = store.getMasterKeys();
- Assert.assertEquals(baseNumKeys, hbaseKeys.length);
-
- // Have to do this last as it will throw an exception
- thrown.expect(NoSuchObjectException.class);
- store.getDatabase(dbNames[1]);
- }
-
- @Test
- public void importOneFunc() throws Exception {
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"onefuncdb1", "onefuncdb2"};
- String[] roles = new String[] {"onefuncrole1", "onefuncrole2"};
- String[] tokenIds = new String[] {"onefunctokenid1", "onefunctokenid2"};
- String[] tokens = new String[] {"onefunctoken1", "onefunctoken2"};
- String[] masterKeys = new String[] {"onefuncmk1", "onefuncmk2"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
- int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
- int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
- int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
- store.getAllTokenIdentifiers().size();
- int baseNumKeys = store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
- // Create the database so I can put the function in it.
- store.createDatabase(
- new Database(dbNames[0], "no description", "file:/tmp", emptyParameters));
-
- HBaseImport importer = new HBaseImport("-f", dbNames[0] + "." + funcNames[0]);
- importer.setConnections(rdbms, store);
- importer.run();
-
- // Make sure there aren't any extra roles
- Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
- Database db = store.getDatabase(dbNames[0]);
- Assert.assertNotNull(db);
-
- Assert.assertEquals(0, store.getAllTables(dbNames[0]).size());
- Assert.assertEquals(1, store.getFunctions(dbNames[0], "*").size());
- Assert.assertNotNull(store.getFunction(dbNames[0], funcNames[0]));
- Assert.assertNull(store.getFunction(dbNames[0], funcNames[1]));
-
- Assert.assertEquals(baseNumDbs + 1, store.getAllDatabases().size());
-
- Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
- String[] hbaseKeys = store.getMasterKeys();
- Assert.assertEquals(baseNumKeys, hbaseKeys.length);
- }
-
- @Test
- public void importOneTableNonPartitioned() throws Exception {
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"onetabdb1", "onetabdb2"};
- String[] roles = new String[] {"onetabrole1", "onetabrole2"};
- String[] tokenIds = new String[] {"onetabtokenid1", "onetabtokenid2"};
- String[] tokens = new String[] {"onetabtoken1", "onetabtoken2"};
- String[] masterKeys = new String[] {"onetabmk1", "onetabmk2"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
- int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
- int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
- int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
- store.getAllTokenIdentifiers().size();
- int baseNumKeys = store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
- // Create the database so I can put the table in it.
- store.createDatabase(
- new Database(dbNames[0], "no description", "file:/tmp", emptyParameters));
-
- HBaseImport importer = new HBaseImport("-t", dbNames[0] + "." + tableNames[0]);
- importer.setConnections(rdbms, store);
- importer.run();
-
- // Make sure there aren't any extra roles
- Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
- Database db = store.getDatabase(dbNames[0]);
- Assert.assertNotNull(db);
-
- Table table = store.getTable(db.getName(), tableNames[0]);
- Assert.assertNotNull(table);
- Assert.assertEquals(1, store.getAllTables(db.getName()).size());
- Assert.assertNull(store.getTable(db.getName(), tableNames[1]));
-
- List<Index> indexes = store.getIndexes(db.getName(), tableNames[0], -1);
- Assert.assertEquals(2, indexes.size());
-
- Assert.assertEquals(0, store.getFunctions(dbNames[0], "*").size());
- Assert.assertEquals(baseNumDbs + 1, store.getAllDatabases().size());
-
- Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
- String[] hbaseKeys = store.getMasterKeys();
- Assert.assertEquals(baseNumKeys, hbaseKeys.length);
- }
-
- @Test
- public void importTablesWithConstraints() throws Exception {
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"onetabwcdb1", "onetabwcdb2"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- setupObjectStore(rdbms, dbNames, now, true);
-
- // Create the database so I can put the table in it.
- store.createDatabase(
- new Database(dbNames[0], "no description", "file:/tmp", emptyParameters));
-
- HBaseImport importer = new HBaseImport("-d", dbNames[0]);
- importer.setConnections(rdbms, store);
- importer.run();
-
- Database db = store.getDatabase(dbNames[0]);
- Assert.assertNotNull(db);
-
- Table table = store.getTable(db.getName(), tableNames[1]);
- Assert.assertNotNull(table);
-
- List<SQLPrimaryKey> pk = store.getPrimaryKeys(dbNames[0], tableNames[1]);
- Assert.assertNotNull(pk);
- Assert.assertEquals(1, pk.size());
- Assert.assertEquals(dbNames[0], pk.get(0).getTable_db());
- Assert.assertEquals(tableNames[1], pk.get(0).getTable_name());
- Assert.assertEquals(0, pk.get(0).getKey_seq());
- Assert.assertEquals("col1", pk.get(0).getColumn_name());
- Assert.assertEquals(dbNames[0] + "_" + pkNames[1], pk.get(0).getPk_name());
- Assert.assertTrue(pk.get(0).isEnable_cstr());
- Assert.assertFalse(pk.get(0).isValidate_cstr());
- Assert.assertTrue(pk.get(0).isRely_cstr());
-
- List<SQLForeignKey> fk =
- store.getForeignKeys(dbNames[0], tableNames[0], dbNames[0], tableNames[1]);
- Assert.assertNotNull(fk);
- Assert.assertEquals(1, fk.size());
- Assert.assertEquals(dbNames[0], fk.get(0).getPktable_db());
- Assert.assertEquals(tableNames[0], fk.get(0).getPktable_name());
- Assert.assertEquals("col1", fk.get(0).getPkcolumn_name());
- Assert.assertEquals(dbNames[0], fk.get(0).getFktable_db());
- Assert.assertEquals(tableNames[1], fk.get(0).getFktable_name());
- Assert.assertEquals("col1", fk.get(0).getFkcolumn_name());
- Assert.assertEquals(0, fk.get(0).getKey_seq());
- Assert.assertEquals(1, fk.get(0).getUpdate_rule());
- Assert.assertEquals(2, fk.get(0).getDelete_rule());
- Assert.assertEquals(dbNames[0] + "_" + fkNames[1], fk.get(0).getFk_name());
- Assert.assertTrue(pk.get(0).isEnable_cstr());
- Assert.assertFalse(pk.get(0).isValidate_cstr());
- Assert.assertTrue(pk.get(0).isRely_cstr());
-
-
- }
-
- @Test
- public void importOneTablePartitioned() throws Exception {
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"onetabpartdb1", "onetabpartodb2"};
- String[] roles = new String[] {"onetabpartorole1", "onetabpartorole2"};
- String[] tokenIds = new String[] {"onetabpartotokenid1", "onetabpartotokenid2"};
- String[] tokens = new String[] {"onetabpartotoken1", "onetabpartotoken2"};
- String[] masterKeys = new String[] {"onetabpartomk1", "onetabpartomk2"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
- int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
- int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
- int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
- store.getAllTokenIdentifiers().size();
- int baseNumKeys = store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
- // Create the database so I can put the table in it.
- store.createDatabase(
- new Database(dbNames[0], "no description", "file:/tmp", emptyParameters));
-
- HBaseImport importer = new HBaseImport("-t", dbNames[0] + "." + tableNames[1]);
- importer.setConnections(rdbms, store);
- importer.run();
-
- // Make sure there aren't any extra roles
- Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
- Database db = store.getDatabase(dbNames[0]);
- Assert.assertNotNull(db);
-
- Table table = store.getTable(db.getName(), tableNames[1]);
- Assert.assertNotNull(table);
- Assert.assertEquals(1, store.getAllTables(db.getName()).size());
-
- for (int j = 0; j < partVals.length; j++) {
- Partition part = store.getPartition(dbNames[0], tableNames[1], Arrays.asList(partVals[j]));
- Assert.assertNotNull(part);
- Assert.assertEquals("file:/tmp/region=" + partVals[j], part.getSd().getLocation());
- }
- Assert.assertEquals(4, store.getPartitions(dbNames[0], tableNames[1], -1).size());
-
- Assert.assertNull(store.getTable(db.getName(), tableNames[0]));
-
- List<Index> indexes = store.getIndexes(db.getName(), tableNames[1], -1);
- Assert.assertEquals(0, indexes.size());
-
- Assert.assertEquals(0, store.getFunctions(dbNames[0], "*").size());
- Assert.assertEquals(baseNumDbs + 1, store.getAllDatabases().size());
-
- Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
- String[] hbaseKeys = store.getMasterKeys();
- Assert.assertEquals(baseNumKeys, hbaseKeys.length);
- }
-
- @Test
- public void importSecurity() throws Exception {
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"securitydb1", "securitydb2"};
- String[] roles = new String[] {"securityrole1", "securityrole2"};
- String[] tokenIds = new String[] {"securitytokenid1", "securitytokenid2"};
- String[] tokens = new String[] {"securitytoken1", "securitytoken2"};
- String[] masterKeys = new String[] {"securitymk1", "securitymk2"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
- int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
- int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
-
- HBaseImport importer = new HBaseImport("-k");
- importer.setConnections(rdbms, store);
- importer.run();
-
- Assert.assertEquals(baseNumRoles, store.listRoleNames().size());
-
- Assert.assertEquals(baseNumDbs, store.getAllDatabases().size());
-
- // I can't test total number of tokens or master keys because the import grabs all and copies
- // them, which means it grabs the ones imported by importAll test (if it's already run).
- // Depending on it already running would make the tests order dependent, which junit doesn't
- // guarantee.
- for (int i = 0; i < tokenIds.length; i++) {
- Assert.assertEquals(tokens[i], store.getToken(tokenIds[i]));
- }
- String[] hbaseKeys = store.getMasterKeys();
- Set<String> keys = new HashSet<>(Arrays.asList(hbaseKeys));
- for (int i = 0; i < masterKeys.length; i++) {
- Assert.assertTrue(keys.contains(masterKeys[i]));
- }
- }
-
- // TODO test for bogus function name
- // TODO test for bogus table name
- // TODO test for non-existent items
-
- @Test
- public void importOneRole() throws Exception {
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"oneroledb1", "oneroledb2"};
- String[] roles = new String[] {"onerolerole1", "onerolerole2"};
- String[] tokenIds = new String[] {"oneroletokenid1", "oneroletokenid2"};
- String[] tokens = new String[] {"oneroletoken1", "oneroletoken2"};
- String[] masterKeys = new String[] {"onerolemk1", "onerolemk2"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now);
-
- int baseNumRoles = store.listRoleNames() == null ? 0 : store.listRoleNames().size();
- int baseNumDbs = store.getAllDatabases() == null ? 0 : store.getAllDatabases().size();
- int baseNumToks = store.getAllTokenIdentifiers() == null ? 0 :
- store.getAllTokenIdentifiers().size();
- int baseNumKeys = store.getMasterKeys() == null ? 0 : store.getMasterKeys().length;
-
- HBaseImport importer = new HBaseImport("-r", roles[0]);
- importer.setConnections(rdbms, store);
- importer.run();
-
- Role role = store.getRole(roles[0]);
- Assert.assertNotNull(role);
- Assert.assertEquals(roles[0], role.getRoleName());
-
- // Make sure there aren't any extra roles
- Assert.assertEquals(baseNumRoles + 1, store.listRoleNames().size());
- Assert.assertEquals(baseNumDbs, store.getAllDatabases().size());
-
- Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size());
- String[] hbaseKeys = store.getMasterKeys();
- Assert.assertEquals(baseNumKeys, hbaseKeys.length);
-
- // Have to do this last as it will throw an exception
- thrown.expect(NoSuchObjectException.class);
- store.getRole(roles[1]);
- }
-
- private void setupObjectStore(RawStore rdbms, String[] roles, String[] dbNames,
- String[] tokenIds, String[] tokens, String[] masterKeys, int now)
- throws MetaException, InvalidObjectException, NoSuchObjectException {
- setupObjectStore(rdbms, roles, dbNames, tokenIds, tokens, masterKeys, now, false);
- }
-
- private void setupObjectStore(RawStore rdbms, String[] dbNames, int now,
- boolean putConstraintsOnTables)
- throws MetaException, InvalidObjectException, NoSuchObjectException {
- setupObjectStore(rdbms, null, dbNames, null, null, null, now, putConstraintsOnTables);
- }
-
- private void setupObjectStore(RawStore rdbms, String[] roles, String[] dbNames,
- String[] tokenIds, String[] tokens, String[] masterKeys, int now,
- boolean putConstraintsOnTables)
- throws MetaException, InvalidObjectException, NoSuchObjectException {
- if (roles != null) {
- for (int i = 0; i < roles.length; i++) {
- rdbms.addRole(roles[i], "me");
- }
- }
-
- for (int i = 0; i < dbNames.length; i++) {
- rdbms.createDatabase(
- new Database(dbNames[i], "no description", "file:/tmp", emptyParameters));
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
- rdbms.createTable(new Table(tableNames[0], dbNames[i], "me", now, now, 0, sd, null,
- emptyParameters, null, null, null));
- if (putConstraintsOnTables) {
- rdbms.addPrimaryKeys(Collections.singletonList(
- new SQLPrimaryKey(dbNames[i], tableNames[0], "col1", 0, dbNames[i] + "_" + pkNames[0],
- true, false, true)
- ));
- }
-
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("region", "string", ""));
- rdbms.createTable(new Table(tableNames[1], dbNames[i], "me", now, now, 0, sd, partCols,
- emptyParameters, null, null, null));
- if (putConstraintsOnTables) {
- rdbms.addPrimaryKeys(Arrays.asList(
- new SQLPrimaryKey(dbNames[i], tableNames[1], "col1", 0, dbNames[i] + "_" + pkNames[1],
- true, false, true)
- ));
- rdbms.addForeignKeys(Collections.singletonList(
- new SQLForeignKey(dbNames[i], tableNames[0], "col1", dbNames[i], tableNames[1],
- "col1", 0, 1, 2, dbNames[i] + "_" + fkNames[1], dbNames[i] + "_" + pkNames[0],
- true, false, true)
- ));
-
- }
-
- for (int j = 0; j < partVals.length; j++) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/region=" + partVals[j]);
- Partition part = new Partition(Arrays.asList(partVals[j]), dbNames[i], tableNames[1],
- now, now, psd, emptyParameters);
- rdbms.addPartition(part);
- }
-
- for (String funcName : funcNames) {
- LOG.debug("Creating new function " + dbNames[i] + "." + funcName);
- rdbms.createFunction(new Function(funcName, dbNames[i], "classname", "ownername",
- PrincipalType.USER, (int) System.currentTimeMillis() / 1000, FunctionType.JAVA,
- Arrays.asList(new ResourceUri(ResourceType.JAR, "uri"))));
- }
-
- for (String indexName : indexNames) {
- LOG.debug("Creating new index " + dbNames[i] + "." + tableNames[0] + "." + indexName);
- String indexTableName = tableNames[0] + "__" + indexName + "__";
- rdbms.createTable(new Table(indexTableName, dbNames[i], "me", now, now, 0, sd, partCols,
- emptyParameters, null, null, null));
- rdbms.addIndex(new Index(indexName, null, dbNames[i], tableNames[0],
- now, now, indexTableName, sd, emptyParameters, false));
- }
- }
- if (tokenIds != null) {
- for (int i = 0; i < tokenIds.length; i++) rdbms.addToken(tokenIds[i], tokens[i]);
- }
- if (masterKeys != null) {
- for (int i = 0; i < masterKeys.length; i++) {
- masterKeySeqs.add(rdbms.addMasterKey(masterKeys[i]));
- }
- }
- }
-
- @Test
- public void parallel() throws Exception {
- int parallelFactor = 10;
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"paralleldb1"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- for (int i = 0; i < dbNames.length; i++) {
- rdbms.createDatabase(
- new Database(dbNames[i], "no description", "file:/tmp", emptyParameters));
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
-
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("region", "string", ""));
- for (int j = 0; j < parallelFactor; j++) {
- rdbms.createTable(new Table("t" + j, dbNames[i], "me", now, now, 0, sd, partCols,
- emptyParameters, null, null, null));
- for (int k = 0; k < parallelFactor; k++) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/region=" + k);
- Partition part = new Partition(Arrays.asList("p" + k), dbNames[i], "t" + j,
- now, now, psd, emptyParameters);
- rdbms.addPartition(part);
- }
- }
- }
-
- HBaseImport importer = new HBaseImport("-p", "2", "-b", "2", "-d", dbNames[0]);
- importer.setConnections(rdbms, store);
- importer.run();
-
- for (int i = 0; i < dbNames.length; i++) {
- Database db = store.getDatabase(dbNames[i]);
- Assert.assertNotNull(db);
-
- for (int j = 0; j < parallelFactor; j++) {
- Table table = store.getTable(db.getName(), "t" + j);
- Assert.assertNotNull(table);
- Assert.assertEquals(now, table.getLastAccessTime());
- Assert.assertEquals("input", table.getSd().getInputFormat());
-
- for (int k = 0; k < parallelFactor; k++) {
- Partition part =
- store.getPartition(dbNames[i], "t" + j, Arrays.asList("p" + k));
- Assert.assertNotNull(part);
- Assert.assertEquals("file:/tmp/region=" + k, part.getSd().getLocation());
- }
-
- Assert.assertEquals(parallelFactor, store.getPartitions(dbNames[i], "t" + j, -1).size());
- }
- Assert.assertEquals(parallelFactor, store.getAllTables(dbNames[i]).size());
-
- }
- }
-
- // Same as the test above except we create 9 of everything instead of 10. This is important
- // because in using a batch size of 2 the previous test guarantees 10 /2 =5 , meaning we'll
- // have 5 writes on the partition queue with exactly 2 entries. In this test we'll handle the
- // case where the last entry in the queue has fewer partitions.
- @Test
- public void parallelOdd() throws Exception {
- int parallelFactor = 9;
- RawStore rdbms;
- rdbms = new ObjectStore();
- rdbms.setConf(conf);
-
- String[] dbNames = new String[] {"oddparalleldb1"};
- int now = (int)System.currentTimeMillis() / 1000;
-
- for (int i = 0; i < dbNames.length; i++) {
- rdbms.createDatabase(
- new Database(dbNames[i], "no description", "file:/tmp", emptyParameters));
-
- List<FieldSchema> cols = new ArrayList<>();
- cols.add(new FieldSchema("col1", "int", "nocomment"));
- SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
- StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
- serde, null, null, emptyParameters);
-
- List<FieldSchema> partCols = new ArrayList<>();
- partCols.add(new FieldSchema("region", "string", ""));
- for (int j = 0; j < parallelFactor; j++) {
- rdbms.createTable(new Table("t" + j, dbNames[i], "me", now, now, 0, sd, partCols,
- emptyParameters, null, null, null));
- for (int k = 0; k < parallelFactor; k++) {
- StorageDescriptor psd = new StorageDescriptor(sd);
- psd.setLocation("file:/tmp/region=" + k);
- Partition part = new Partition(Arrays.asList("p" + k), dbNames[i], "t" + j,
- now, now, psd, emptyParameters);
- rdbms.addPartition(part);
- }
- }
- }
-
- HBaseImport importer = new HBaseImport("-p", "2", "-b", "2", "-d", dbNames[0]);
- importer.setConnections(rdbms, store);
- importer.run();
-
- for (int i = 0; i < dbNames.length; i++) {
- Database db = store.getDatabase(dbNames[i]);
- Assert.assertNotNull(db);
-
- for (int j = 0; j < parallelFactor; j++) {
- Table table = store.getTable(db.getName(), "t" + j);
- Assert.assertNotNull(table);
- Assert.assertEquals(now, table.getLastAccessTime());
- Assert.assertEquals("input", table.getSd().getInputFormat());
-
- for (int k = 0; k < parallelFactor; k++) {
- Partition part =
- store.getPartition(dbNames[i], "t" + j, Arrays.asList("p" + k));
- Assert.assertNotNull(part);
- Assert.assertEquals("file:/tmp/region=" + k, part.getSd().getLocation());
- }
-
- Assert.assertEquals(parallelFactor, store.getPartitions(dbNames[i], "t" + j, -1).size());
- }
- Assert.assertEquals(parallelFactor, store.getAllTables(dbNames[i]).size());
-
- }
- }
-}
[13/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/hbase/HbaseMetastoreProto.java
----------------------------------------------------------------------
diff --git a/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/hbase/HbaseMetastoreProto.java b/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/hbase/HbaseMetastoreProto.java
deleted file mode 100644
index 9cf1ee2..0000000
--- a/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/hbase/HbaseMetastoreProto.java
+++ /dev/null
@@ -1,46709 +0,0 @@
-// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: hbase_metastore_proto.proto
-
-package org.apache.hadoop.hive.metastore.hbase;
-
-public final class HbaseMetastoreProto {
- private HbaseMetastoreProto() {}
- public static void registerAllExtensions(
- com.google.protobuf.ExtensionRegistry registry) {
- }
- /**
- * Protobuf enum {@code org.apache.hadoop.hive.metastore.hbase.PrincipalType}
- */
- public enum PrincipalType
- implements com.google.protobuf.ProtocolMessageEnum {
- /**
- * <code>USER = 0;</code>
- */
- USER(0, 0),
- /**
- * <code>ROLE = 1;</code>
- */
- ROLE(1, 1),
- ;
-
- /**
- * <code>USER = 0;</code>
- */
- public static final int USER_VALUE = 0;
- /**
- * <code>ROLE = 1;</code>
- */
- public static final int ROLE_VALUE = 1;
-
-
- public final int getNumber() { return value; }
-
- public static PrincipalType valueOf(int value) {
- switch (value) {
- case 0: return USER;
- case 1: return ROLE;
- default: return null;
- }
- }
-
- public static com.google.protobuf.Internal.EnumLiteMap<PrincipalType>
- internalGetValueMap() {
- return internalValueMap;
- }
- private static com.google.protobuf.Internal.EnumLiteMap<PrincipalType>
- internalValueMap =
- new com.google.protobuf.Internal.EnumLiteMap<PrincipalType>() {
- public PrincipalType findValueByNumber(int number) {
- return PrincipalType.valueOf(number);
- }
- };
-
- public final com.google.protobuf.Descriptors.EnumValueDescriptor
- getValueDescriptor() {
- return getDescriptor().getValues().get(index);
- }
- public final com.google.protobuf.Descriptors.EnumDescriptor
- getDescriptorForType() {
- return getDescriptor();
- }
- public static final com.google.protobuf.Descriptors.EnumDescriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.getDescriptor().getEnumTypes().get(0);
- }
-
- private static final PrincipalType[] VALUES = values();
-
- public static PrincipalType valueOf(
- com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
- if (desc.getType() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "EnumValueDescriptor is not for this type.");
- }
- return VALUES[desc.getIndex()];
- }
-
- private final int index;
- private final int value;
-
- private PrincipalType(int index, int value) {
- this.index = index;
- this.value = value;
- }
-
- // @@protoc_insertion_point(enum_scope:org.apache.hadoop.hive.metastore.hbase.PrincipalType)
- }
-
- public interface AggrStatsOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // required int64 parts_found = 1;
- /**
- * <code>required int64 parts_found = 1;</code>
- */
- boolean hasPartsFound();
- /**
- * <code>required int64 parts_found = 1;</code>
- */
- long getPartsFound();
-
- // repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats>
- getColStatsList();
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats getColStats(int index);
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- int getColStatsCount();
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- java.util.List<? extends org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder>
- getColStatsOrBuilderList();
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder getColStatsOrBuilder(
- int index);
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStats}
- */
- public static final class AggrStats extends
- com.google.protobuf.GeneratedMessage
- implements AggrStatsOrBuilder {
- // Use AggrStats.newBuilder() to construct.
- private AggrStats(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private AggrStats(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final AggrStats defaultInstance;
- public static AggrStats getDefaultInstance() {
- return defaultInstance;
- }
-
- public AggrStats getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private AggrStats(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 8: {
- bitField0_ |= 0x00000001;
- partsFound_ = input.readInt64();
- break;
- }
- case 18: {
- if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
- colStats_ = new java.util.ArrayList<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats>();
- mutable_bitField0_ |= 0x00000002;
- }
- colStats_.add(input.readMessage(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.PARSER, extensionRegistry));
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) {
- colStats_ = java.util.Collections.unmodifiableList(colStats_);
- }
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.Builder.class);
- }
-
- public static com.google.protobuf.Parser<AggrStats> PARSER =
- new com.google.protobuf.AbstractParser<AggrStats>() {
- public AggrStats parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new AggrStats(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<AggrStats> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // required int64 parts_found = 1;
- public static final int PARTS_FOUND_FIELD_NUMBER = 1;
- private long partsFound_;
- /**
- * <code>required int64 parts_found = 1;</code>
- */
- public boolean hasPartsFound() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required int64 parts_found = 1;</code>
- */
- public long getPartsFound() {
- return partsFound_;
- }
-
- // repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;
- public static final int COL_STATS_FIELD_NUMBER = 2;
- private java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> colStats_;
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> getColStatsList() {
- return colStats_;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public java.util.List<? extends org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder>
- getColStatsOrBuilderList() {
- return colStats_;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public int getColStatsCount() {
- return colStats_.size();
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats getColStats(int index) {
- return colStats_.get(index);
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder getColStatsOrBuilder(
- int index) {
- return colStats_.get(index);
- }
-
- private void initFields() {
- partsFound_ = 0L;
- colStats_ = java.util.Collections.emptyList();
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- if (!hasPartsFound()) {
- memoizedIsInitialized = 0;
- return false;
- }
- for (int i = 0; i < getColStatsCount(); i++) {
- if (!getColStats(i).isInitialized()) {
- memoizedIsInitialized = 0;
- return false;
- }
- }
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeInt64(1, partsFound_);
- }
- for (int i = 0; i < colStats_.size(); i++) {
- output.writeMessage(2, colStats_.get(i));
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(1, partsFound_);
- }
- for (int i = 0; i < colStats_.size(); i++) {
- size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(2, colStats_.get(i));
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStats}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- getColStatsFieldBuilder();
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- partsFound_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000001);
- if (colStatsBuilder_ == null) {
- colStats_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000002);
- } else {
- colStatsBuilder_.clear();
- }
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStats_descriptor;
- }
-
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats getDefaultInstanceForType() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats build() {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats buildPartial() {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats result = new org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.partsFound_ = partsFound_;
- if (colStatsBuilder_ == null) {
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- colStats_ = java.util.Collections.unmodifiableList(colStats_);
- bitField0_ = (bitField0_ & ~0x00000002);
- }
- result.colStats_ = colStats_;
- } else {
- result.colStats_ = colStatsBuilder_.build();
- }
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats) {
- return mergeFrom((org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats other) {
- if (other == org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats.getDefaultInstance()) return this;
- if (other.hasPartsFound()) {
- setPartsFound(other.getPartsFound());
- }
- if (colStatsBuilder_ == null) {
- if (!other.colStats_.isEmpty()) {
- if (colStats_.isEmpty()) {
- colStats_ = other.colStats_;
- bitField0_ = (bitField0_ & ~0x00000002);
- } else {
- ensureColStatsIsMutable();
- colStats_.addAll(other.colStats_);
- }
- onChanged();
- }
- } else {
- if (!other.colStats_.isEmpty()) {
- if (colStatsBuilder_.isEmpty()) {
- colStatsBuilder_.dispose();
- colStatsBuilder_ = null;
- colStats_ = other.colStats_;
- bitField0_ = (bitField0_ & ~0x00000002);
- colStatsBuilder_ =
- com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
- getColStatsFieldBuilder() : null;
- } else {
- colStatsBuilder_.addAllMessages(other.colStats_);
- }
- }
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- if (!hasPartsFound()) {
-
- return false;
- }
- for (int i = 0; i < getColStatsCount(); i++) {
- if (!getColStats(i).isInitialized()) {
-
- return false;
- }
- }
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStats) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // required int64 parts_found = 1;
- private long partsFound_ ;
- /**
- * <code>required int64 parts_found = 1;</code>
- */
- public boolean hasPartsFound() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required int64 parts_found = 1;</code>
- */
- public long getPartsFound() {
- return partsFound_;
- }
- /**
- * <code>required int64 parts_found = 1;</code>
- */
- public Builder setPartsFound(long value) {
- bitField0_ |= 0x00000001;
- partsFound_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int64 parts_found = 1;</code>
- */
- public Builder clearPartsFound() {
- bitField0_ = (bitField0_ & ~0x00000001);
- partsFound_ = 0L;
- onChanged();
- return this;
- }
-
- // repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;
- private java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> colStats_ =
- java.util.Collections.emptyList();
- private void ensureColStatsIsMutable() {
- if (!((bitField0_ & 0x00000002) == 0x00000002)) {
- colStats_ = new java.util.ArrayList<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats>(colStats_);
- bitField0_ |= 0x00000002;
- }
- }
-
- private com.google.protobuf.RepeatedFieldBuilder<
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder> colStatsBuilder_;
-
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> getColStatsList() {
- if (colStatsBuilder_ == null) {
- return java.util.Collections.unmodifiableList(colStats_);
- } else {
- return colStatsBuilder_.getMessageList();
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public int getColStatsCount() {
- if (colStatsBuilder_ == null) {
- return colStats_.size();
- } else {
- return colStatsBuilder_.getCount();
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats getColStats(int index) {
- if (colStatsBuilder_ == null) {
- return colStats_.get(index);
- } else {
- return colStatsBuilder_.getMessage(index);
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public Builder setColStats(
- int index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats value) {
- if (colStatsBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureColStatsIsMutable();
- colStats_.set(index, value);
- onChanged();
- } else {
- colStatsBuilder_.setMessage(index, value);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public Builder setColStats(
- int index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder builderForValue) {
- if (colStatsBuilder_ == null) {
- ensureColStatsIsMutable();
- colStats_.set(index, builderForValue.build());
- onChanged();
- } else {
- colStatsBuilder_.setMessage(index, builderForValue.build());
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public Builder addColStats(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats value) {
- if (colStatsBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureColStatsIsMutable();
- colStats_.add(value);
- onChanged();
- } else {
- colStatsBuilder_.addMessage(value);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public Builder addColStats(
- int index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats value) {
- if (colStatsBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureColStatsIsMutable();
- colStats_.add(index, value);
- onChanged();
- } else {
- colStatsBuilder_.addMessage(index, value);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public Builder addColStats(
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder builderForValue) {
- if (colStatsBuilder_ == null) {
- ensureColStatsIsMutable();
- colStats_.add(builderForValue.build());
- onChanged();
- } else {
- colStatsBuilder_.addMessage(builderForValue.build());
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public Builder addColStats(
- int index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder builderForValue) {
- if (colStatsBuilder_ == null) {
- ensureColStatsIsMutable();
- colStats_.add(index, builderForValue.build());
- onChanged();
- } else {
- colStatsBuilder_.addMessage(index, builderForValue.build());
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public Builder addAllColStats(
- java.lang.Iterable<? extends org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats> values) {
- if (colStatsBuilder_ == null) {
- ensureColStatsIsMutable();
- super.addAll(values, colStats_);
- onChanged();
- } else {
- colStatsBuilder_.addAllMessages(values);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public Builder clearColStats() {
- if (colStatsBuilder_ == null) {
- colStats_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000002);
- onChanged();
- } else {
- colStatsBuilder_.clear();
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public Builder removeColStats(int index) {
- if (colStatsBuilder_ == null) {
- ensureColStatsIsMutable();
- colStats_.remove(index);
- onChanged();
- } else {
- colStatsBuilder_.remove(index);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder getColStatsBuilder(
- int index) {
- return getColStatsFieldBuilder().getBuilder(index);
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder getColStatsOrBuilder(
- int index) {
- if (colStatsBuilder_ == null) {
- return colStats_.get(index); } else {
- return colStatsBuilder_.getMessageOrBuilder(index);
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public java.util.List<? extends org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder>
- getColStatsOrBuilderList() {
- if (colStatsBuilder_ != null) {
- return colStatsBuilder_.getMessageOrBuilderList();
- } else {
- return java.util.Collections.unmodifiableList(colStats_);
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder addColStatsBuilder() {
- return getColStatsFieldBuilder().addBuilder(
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.getDefaultInstance());
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder addColStatsBuilder(
- int index) {
- return getColStatsFieldBuilder().addBuilder(
- index, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.getDefaultInstance());
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.hbase.ColumnStats col_stats = 2;</code>
- */
- public java.util.List<org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder>
- getColStatsBuilderList() {
- return getColStatsFieldBuilder().getBuilderList();
- }
- private com.google.protobuf.RepeatedFieldBuilder<
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder>
- getColStatsFieldBuilder() {
- if (colStatsBuilder_ == null) {
- colStatsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStats.Builder, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.ColumnStatsOrBuilder>(
- colStats_,
- ((bitField0_ & 0x00000002) == 0x00000002),
- getParentForChildren(),
- isClean());
- colStats_ = null;
- }
- return colStatsBuilder_;
- }
-
- // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.metastore.hbase.AggrStats)
- }
-
- static {
- defaultInstance = new AggrStats(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.metastore.hbase.AggrStats)
- }
-
- public interface AggrStatsBloomFilterOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // required bytes db_name = 1;
- /**
- * <code>required bytes db_name = 1;</code>
- */
- boolean hasDbName();
- /**
- * <code>required bytes db_name = 1;</code>
- */
- com.google.protobuf.ByteString getDbName();
-
- // required bytes table_name = 2;
- /**
- * <code>required bytes table_name = 2;</code>
- */
- boolean hasTableName();
- /**
- * <code>required bytes table_name = 2;</code>
- */
- com.google.protobuf.ByteString getTableName();
-
- // required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- boolean hasBloomFilter();
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter getBloomFilter();
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilterOrBuilder getBloomFilterOrBuilder();
-
- // required int64 aggregated_at = 4;
- /**
- * <code>required int64 aggregated_at = 4;</code>
- */
- boolean hasAggregatedAt();
- /**
- * <code>required int64 aggregated_at = 4;</code>
- */
- long getAggregatedAt();
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter}
- */
- public static final class AggrStatsBloomFilter extends
- com.google.protobuf.GeneratedMessage
- implements AggrStatsBloomFilterOrBuilder {
- // Use AggrStatsBloomFilter.newBuilder() to construct.
- private AggrStatsBloomFilter(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private AggrStatsBloomFilter(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final AggrStatsBloomFilter defaultInstance;
- public static AggrStatsBloomFilter getDefaultInstance() {
- return defaultInstance;
- }
-
- public AggrStatsBloomFilter getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private AggrStatsBloomFilter(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- bitField0_ |= 0x00000001;
- dbName_ = input.readBytes();
- break;
- }
- case 18: {
- bitField0_ |= 0x00000002;
- tableName_ = input.readBytes();
- break;
- }
- case 26: {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder subBuilder = null;
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- subBuilder = bloomFilter_.toBuilder();
- }
- bloomFilter_ = input.readMessage(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.PARSER, extensionRegistry);
- if (subBuilder != null) {
- subBuilder.mergeFrom(bloomFilter_);
- bloomFilter_ = subBuilder.buildPartial();
- }
- bitField0_ |= 0x00000004;
- break;
- }
- case 32: {
- bitField0_ |= 0x00000008;
- aggregatedAt_ = input.readInt64();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.Builder.class);
- }
-
- public static com.google.protobuf.Parser<AggrStatsBloomFilter> PARSER =
- new com.google.protobuf.AbstractParser<AggrStatsBloomFilter>() {
- public AggrStatsBloomFilter parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new AggrStatsBloomFilter(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<AggrStatsBloomFilter> getParserForType() {
- return PARSER;
- }
-
- public interface BloomFilterOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // required int32 num_bits = 1;
- /**
- * <code>required int32 num_bits = 1;</code>
- */
- boolean hasNumBits();
- /**
- * <code>required int32 num_bits = 1;</code>
- */
- int getNumBits();
-
- // required int32 num_funcs = 2;
- /**
- * <code>required int32 num_funcs = 2;</code>
- */
- boolean hasNumFuncs();
- /**
- * <code>required int32 num_funcs = 2;</code>
- */
- int getNumFuncs();
-
- // repeated int64 bits = 3;
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- java.util.List<java.lang.Long> getBitsList();
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- int getBitsCount();
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- long getBits(int index);
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter}
- */
- public static final class BloomFilter extends
- com.google.protobuf.GeneratedMessage
- implements BloomFilterOrBuilder {
- // Use BloomFilter.newBuilder() to construct.
- private BloomFilter(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private BloomFilter(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final BloomFilter defaultInstance;
- public static BloomFilter getDefaultInstance() {
- return defaultInstance;
- }
-
- public BloomFilter getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private BloomFilter(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 8: {
- bitField0_ |= 0x00000001;
- numBits_ = input.readInt32();
- break;
- }
- case 16: {
- bitField0_ |= 0x00000002;
- numFuncs_ = input.readInt32();
- break;
- }
- case 24: {
- if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
- bits_ = new java.util.ArrayList<java.lang.Long>();
- mutable_bitField0_ |= 0x00000004;
- }
- bits_.add(input.readInt64());
- break;
- }
- case 26: {
- int length = input.readRawVarint32();
- int limit = input.pushLimit(length);
- if (!((mutable_bitField0_ & 0x00000004) == 0x00000004) && input.getBytesUntilLimit() > 0) {
- bits_ = new java.util.ArrayList<java.lang.Long>();
- mutable_bitField0_ |= 0x00000004;
- }
- while (input.getBytesUntilLimit() > 0) {
- bits_.add(input.readInt64());
- }
- input.popLimit(limit);
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
- bits_ = java.util.Collections.unmodifiableList(bits_);
- }
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder.class);
- }
-
- public static com.google.protobuf.Parser<BloomFilter> PARSER =
- new com.google.protobuf.AbstractParser<BloomFilter>() {
- public BloomFilter parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new BloomFilter(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<BloomFilter> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // required int32 num_bits = 1;
- public static final int NUM_BITS_FIELD_NUMBER = 1;
- private int numBits_;
- /**
- * <code>required int32 num_bits = 1;</code>
- */
- public boolean hasNumBits() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required int32 num_bits = 1;</code>
- */
- public int getNumBits() {
- return numBits_;
- }
-
- // required int32 num_funcs = 2;
- public static final int NUM_FUNCS_FIELD_NUMBER = 2;
- private int numFuncs_;
- /**
- * <code>required int32 num_funcs = 2;</code>
- */
- public boolean hasNumFuncs() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>required int32 num_funcs = 2;</code>
- */
- public int getNumFuncs() {
- return numFuncs_;
- }
-
- // repeated int64 bits = 3;
- public static final int BITS_FIELD_NUMBER = 3;
- private java.util.List<java.lang.Long> bits_;
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public java.util.List<java.lang.Long>
- getBitsList() {
- return bits_;
- }
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public int getBitsCount() {
- return bits_.size();
- }
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public long getBits(int index) {
- return bits_.get(index);
- }
-
- private void initFields() {
- numBits_ = 0;
- numFuncs_ = 0;
- bits_ = java.util.Collections.emptyList();
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- if (!hasNumBits()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasNumFuncs()) {
- memoizedIsInitialized = 0;
- return false;
- }
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeInt32(1, numBits_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeInt32(2, numFuncs_);
- }
- for (int i = 0; i < bits_.size(); i++) {
- output.writeInt64(3, bits_.get(i));
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt32Size(1, numBits_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt32Size(2, numFuncs_);
- }
- {
- int dataSize = 0;
- for (int i = 0; i < bits_.size(); i++) {
- dataSize += com.google.protobuf.CodedOutputStream
- .computeInt64SizeNoTag(bits_.get(i));
- }
- size += dataSize;
- size += 1 * getBitsList().size();
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilterOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- numBits_ = 0;
- bitField0_ = (bitField0_ & ~0x00000001);
- numFuncs_ = 0;
- bitField0_ = (bitField0_ & ~0x00000002);
- bits_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000004);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_BloomFilter_descriptor;
- }
-
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter getDefaultInstanceForType() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter build() {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter buildPartial() {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter result = new org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.numBits_ = numBits_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- result.numFuncs_ = numFuncs_;
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- bits_ = java.util.Collections.unmodifiableList(bits_);
- bitField0_ = (bitField0_ & ~0x00000004);
- }
- result.bits_ = bits_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter) {
- return mergeFrom((org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter other) {
- if (other == org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance()) return this;
- if (other.hasNumBits()) {
- setNumBits(other.getNumBits());
- }
- if (other.hasNumFuncs()) {
- setNumFuncs(other.getNumFuncs());
- }
- if (!other.bits_.isEmpty()) {
- if (bits_.isEmpty()) {
- bits_ = other.bits_;
- bitField0_ = (bitField0_ & ~0x00000004);
- } else {
- ensureBitsIsMutable();
- bits_.addAll(other.bits_);
- }
- onChanged();
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- if (!hasNumBits()) {
-
- return false;
- }
- if (!hasNumFuncs()) {
-
- return false;
- }
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // required int32 num_bits = 1;
- private int numBits_ ;
- /**
- * <code>required int32 num_bits = 1;</code>
- */
- public boolean hasNumBits() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required int32 num_bits = 1;</code>
- */
- public int getNumBits() {
- return numBits_;
- }
- /**
- * <code>required int32 num_bits = 1;</code>
- */
- public Builder setNumBits(int value) {
- bitField0_ |= 0x00000001;
- numBits_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int32 num_bits = 1;</code>
- */
- public Builder clearNumBits() {
- bitField0_ = (bitField0_ & ~0x00000001);
- numBits_ = 0;
- onChanged();
- return this;
- }
-
- // required int32 num_funcs = 2;
- private int numFuncs_ ;
- /**
- * <code>required int32 num_funcs = 2;</code>
- */
- public boolean hasNumFuncs() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>required int32 num_funcs = 2;</code>
- */
- public int getNumFuncs() {
- return numFuncs_;
- }
- /**
- * <code>required int32 num_funcs = 2;</code>
- */
- public Builder setNumFuncs(int value) {
- bitField0_ |= 0x00000002;
- numFuncs_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int32 num_funcs = 2;</code>
- */
- public Builder clearNumFuncs() {
- bitField0_ = (bitField0_ & ~0x00000002);
- numFuncs_ = 0;
- onChanged();
- return this;
- }
-
- // repeated int64 bits = 3;
- private java.util.List<java.lang.Long> bits_ = java.util.Collections.emptyList();
- private void ensureBitsIsMutable() {
- if (!((bitField0_ & 0x00000004) == 0x00000004)) {
- bits_ = new java.util.ArrayList<java.lang.Long>(bits_);
- bitField0_ |= 0x00000004;
- }
- }
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public java.util.List<java.lang.Long>
- getBitsList() {
- return java.util.Collections.unmodifiableList(bits_);
- }
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public int getBitsCount() {
- return bits_.size();
- }
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public long getBits(int index) {
- return bits_.get(index);
- }
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public Builder setBits(
- int index, long value) {
- ensureBitsIsMutable();
- bits_.set(index, value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public Builder addBits(long value) {
- ensureBitsIsMutable();
- bits_.add(value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public Builder addAllBits(
- java.lang.Iterable<? extends java.lang.Long> values) {
- ensureBitsIsMutable();
- super.addAll(values, bits_);
- onChanged();
- return this;
- }
- /**
- * <code>repeated int64 bits = 3;</code>
- */
- public Builder clearBits() {
- bits_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000004);
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter)
- }
-
- static {
- defaultInstance = new BloomFilter(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter)
- }
-
- private int bitField0_;
- // required bytes db_name = 1;
- public static final int DB_NAME_FIELD_NUMBER = 1;
- private com.google.protobuf.ByteString dbName_;
- /**
- * <code>required bytes db_name = 1;</code>
- */
- public boolean hasDbName() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required bytes db_name = 1;</code>
- */
- public com.google.protobuf.ByteString getDbName() {
- return dbName_;
- }
-
- // required bytes table_name = 2;
- public static final int TABLE_NAME_FIELD_NUMBER = 2;
- private com.google.protobuf.ByteString tableName_;
- /**
- * <code>required bytes table_name = 2;</code>
- */
- public boolean hasTableName() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>required bytes table_name = 2;</code>
- */
- public com.google.protobuf.ByteString getTableName() {
- return tableName_;
- }
-
- // required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;
- public static final int BLOOM_FILTER_FIELD_NUMBER = 3;
- private org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter bloomFilter_;
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- public boolean hasBloomFilter() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter getBloomFilter() {
- return bloomFilter_;
- }
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilterOrBuilder getBloomFilterOrBuilder() {
- return bloomFilter_;
- }
-
- // required int64 aggregated_at = 4;
- public static final int AGGREGATED_AT_FIELD_NUMBER = 4;
- private long aggregatedAt_;
- /**
- * <code>required int64 aggregated_at = 4;</code>
- */
- public boolean hasAggregatedAt() {
- return ((bitField0_ & 0x00000008) == 0x00000008);
- }
- /**
- * <code>required int64 aggregated_at = 4;</code>
- */
- public long getAggregatedAt() {
- return aggregatedAt_;
- }
-
- private void initFields() {
- dbName_ = com.google.protobuf.ByteString.EMPTY;
- tableName_ = com.google.protobuf.ByteString.EMPTY;
- bloomFilter_ = org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
- aggregatedAt_ = 0L;
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- if (!hasDbName()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasTableName()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasBloomFilter()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasAggregatedAt()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!getBloomFilter().isInitialized()) {
- memoizedIsInitialized = 0;
- return false;
- }
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, dbName_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeBytes(2, tableName_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeMessage(3, bloomFilter_);
- }
- if (((bitField0_ & 0x00000008) == 0x00000008)) {
- output.writeInt64(4, aggregatedAt_);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, dbName_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(2, tableName_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(3, bloomFilter_);
- }
- if (((bitField0_ & 0x00000008) == 0x00000008)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(4, aggregatedAt_);
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilterOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.class, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- getBloomFilterFieldBuilder();
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- dbName_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000001);
- tableName_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000002);
- if (bloomFilterBuilder_ == null) {
- bloomFilter_ = org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
- } else {
- bloomFilterBuilder_.clear();
- }
- bitField0_ = (bitField0_ & ~0x00000004);
- aggregatedAt_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000008);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.internal_static_org_apache_hadoop_hive_metastore_hbase_AggrStatsBloomFilter_descriptor;
- }
-
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter getDefaultInstanceForType() {
- return org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter build() {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter buildPartial() {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter result = new org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.dbName_ = dbName_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- result.tableName_ = tableName_;
- if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
- to_bitField0_ |= 0x00000004;
- }
- if (bloomFilterBuilder_ == null) {
- result.bloomFilter_ = bloomFilter_;
- } else {
- result.bloomFilter_ = bloomFilterBuilder_.build();
- }
- if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
- to_bitField0_ |= 0x00000008;
- }
- result.aggregatedAt_ = aggregatedAt_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter) {
- return mergeFrom((org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter other) {
- if (other == org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.getDefaultInstance()) return this;
- if (other.hasDbName()) {
- setDbName(other.getDbName());
- }
- if (other.hasTableName()) {
- setTableName(other.getTableName());
- }
- if (other.hasBloomFilter()) {
- mergeBloomFilter(other.getBloomFilter());
- }
- if (other.hasAggregatedAt()) {
- setAggregatedAt(other.getAggregatedAt());
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- if (!hasDbName()) {
-
- return false;
- }
- if (!hasTableName()) {
-
- return false;
- }
- if (!hasBloomFilter()) {
-
- return false;
- }
- if (!hasAggregatedAt()) {
-
- return false;
- }
- if (!getBloomFilter().isInitialized()) {
-
- return false;
- }
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // required bytes db_name = 1;
- private com.google.protobuf.ByteString dbName_ = com.google.protobuf.ByteString.EMPTY;
- /**
- * <code>required bytes db_name = 1;</code>
- */
- public boolean hasDbName() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required bytes db_name = 1;</code>
- */
- public com.google.protobuf.ByteString getDbName() {
- return dbName_;
- }
- /**
- * <code>required bytes db_name = 1;</code>
- */
- public Builder setDbName(com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- dbName_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required bytes db_name = 1;</code>
- */
- public Builder clearDbName() {
- bitField0_ = (bitField0_ & ~0x00000001);
- dbName_ = getDefaultInstance().getDbName();
- onChanged();
- return this;
- }
-
- // required bytes table_name = 2;
- private com.google.protobuf.ByteString tableName_ = com.google.protobuf.ByteString.EMPTY;
- /**
- * <code>required bytes table_name = 2;</code>
- */
- public boolean hasTableName() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>required bytes table_name = 2;</code>
- */
- public com.google.protobuf.ByteString getTableName() {
- return tableName_;
- }
- /**
- * <code>required bytes table_name = 2;</code>
- */
- public Builder setTableName(com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000002;
- tableName_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required bytes table_name = 2;</code>
- */
- public Builder clearTableName() {
- bitField0_ = (bitField0_ & ~0x00000002);
- tableName_ = getDefaultInstance().getTableName();
- onChanged();
- return this;
- }
-
- // required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;
- private org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter bloomFilter_ = org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
- private com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder, org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilterOrBuilder> bloomFilterBuilder_;
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- public boolean hasBloomFilter() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- public org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter getBloomFilter() {
- if (bloomFilterBuilder_ == null) {
- return bloomFilter_;
- } else {
- return bloomFilterBuilder_.getMessage();
- }
- }
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- public Builder setBloomFilter(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter value) {
- if (bloomFilterBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- bloomFilter_ = value;
- onChanged();
- } else {
- bloomFilterBuilder_.setMessage(value);
- }
- bitField0_ |= 0x00000004;
- return this;
- }
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- public Builder setBloomFilter(
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.Builder builderForValue) {
- if (bloomFilterBuilder_ == null) {
- bloomFilter_ = builderForValue.build();
- onChanged();
- } else {
- bloomFilterBuilder_.setMessage(builderForValue.build());
- }
- bitField0_ |= 0x00000004;
- return this;
- }
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- public Builder mergeBloomFilter(org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter value) {
- if (bloomFilterBuilder_ == null) {
- if (((bitField0_ & 0x00000004) == 0x00000004) &&
- bloomFilter_ != org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance()) {
- bloomFilter_ =
- org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.newBuilder(bloomFilter_).mergeFrom(value).buildPartial();
- } else {
- bloomFilter_ = value;
- }
- onChanged();
- } else {
- bloomFilterBuilder_.mergeFrom(value);
- }
- bitField0_ |= 0x00000004;
- return this;
- }
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter bloom_filter = 3;</code>
- */
- public Builder clearBloomFilter() {
- if (bloomFilterBuilder_ == null) {
- bloomFilter_ = org.apache.hadoop.hive.metastore.hbase.HbaseMetastoreProto.AggrStatsBloomFilter.BloomFilter.getDefaultInstance();
- onChanged();
- } else {
- bloomFilterBuilder_.clear();
- }
- bitField0_ = (bitField0_ & ~0x00000004);
- return this;
- }
- /**
- * <code>required .org.apache.hadoop.hive.metastore.hbase.AggrStatsBloomFilter.BloomFilter blo
<TRUNCATED>