You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2015/12/09 20:32:58 UTC
hive git commit: HIVE-12499 : Add HMS metrics for number of tables
and partitions (Szehon, reviewed by Jimmy Xiang)
Repository: hive
Updated Branches:
refs/heads/master 695bde3ac -> 546c75c59
HIVE-12499 : Add HMS metrics for number of tables and partitions (Szehon, reviewed by Jimmy Xiang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/546c75c5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/546c75c5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/546c75c5
Branch: refs/heads/master
Commit: 546c75c598617d79eae66bdb33c92106b486ad24
Parents: 695bde3
Author: Szehon Ho <sz...@cloudera.com>
Authored: Wed Dec 9 11:32:21 2015 -0800
Committer: Szehon Ho <sz...@cloudera.com>
Committed: Wed Dec 9 11:32:21 2015 -0800
----------------------------------------------------------------------
.../common/metrics/common/MetricsConstant.java | 8 ++
.../hive/common/metrics/MetricsTestUtils.java | 4 +-
.../hive/metastore/TestMetaStoreMetrics.java | 83 +++++++++--
.../hbase/TestHBaseMetastoreMetrics.java | 136 +++++++++++++++++++
.../hive/metastore/HMSMetricsListener.java | 113 +++++++++++++++
.../hadoop/hive/metastore/HiveMetaStore.java | 51 +++++++
.../hadoop/hive/metastore/ObjectStore.java | 38 +++++-
.../apache/hadoop/hive/metastore/RawStore.java | 19 +++
.../hive/metastore/hbase/HBaseReadWrite.java | 20 +++
.../hadoop/hive/metastore/hbase/HBaseStore.java | 30 ++++
.../DummyRawStoreControlledCommit.java | 15 ++
.../DummyRawStoreForJdoConnection.java | 15 ++
12 files changed, 520 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
index 95e2bcf..3477187 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java
@@ -39,4 +39,12 @@ public class MetricsConstant {
public static final String OPERATION_PREFIX = "hs2_operation_";
public static final String COMPLETED_OPERATION_PREFIX = "hs2_completed_operation_";
+
+ public static final String INIT_TOTAL_DATABASES = "init_total_count_dbs";
+ public static final String INIT_TOTAL_TABLES = "init_total_count_tables";
+ public static final String INIT_TOTAL_PARTITIONS = "init_total_count_partitions";
+
+ public static final String DELTA_TOTAL_DATABASES = "delta_total_count_dbs";
+ public static final String DELTA_TOTAL_TABLES = "delta_total_count_tables";
+ public static final String DELTA_TOTAL_PARTITIONS = "delta_total_count_partitions";
}
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
index fd420f7..f21b431 100644
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java
@@ -45,9 +45,9 @@ public class MetricsTestUtils {
}
public static void verifyMetricFile(File jsonReportFile, MetricsCategory category, String metricsName,
- Object value) throws Exception {
+ Object expectedValue) throws Exception {
JsonNode jsonNode = getJsonNode(jsonReportFile, category, metricsName);
- Assert.assertEquals(jsonNode.asText(), value.toString());
+ Assert.assertEquals(expectedValue.toString(), jsonNode.asText());
}
private static JsonNode getJsonNode(File jsonReportFile, MetricsCategory category, String metricsName) throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
index f571c7c..bbfee1d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java
@@ -19,26 +19,32 @@ package org.apache.hadoop.hive.metastore;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
-import junit.framework.TestCase;
import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.server.HiveServer2;
-import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
-import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
@@ -71,19 +77,18 @@ public class TestMetaStoreMetrics {
hiveConf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf);
-
SessionState.start(new CliSessionState(hiveConf));
driver = new Driver(hiveConf);
}
+
@Test
- public void testMetricsFile() throws Exception {
+ public void testMethodCounts() throws Exception {
driver.run("show databases");
//give timer thread a chance to print the metrics
Thread.sleep(2000);
- //As the file is being written, try a few times.
//This can be replaced by CodahaleMetrics's JsonServlet reporter once it is exposed.
byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath()));
ObjectMapper objectMapper = new ObjectMapper();
@@ -95,6 +100,66 @@ public class TestMetaStoreMetrics {
Assert.assertTrue(methodCountNode.asInt() > 0);
}
+ @Test
+ public void testMetaDataCounts() throws Exception {
+ //1 databases created
+ driver.run("create database testdb1");
+
+ //4 tables
+ driver.run("create table testtbl1 (key string)");
+ driver.run("create table testtblpart (key string) partitioned by (partkey string)");
+ driver.run("use testdb1");
+ driver.run("create table testtbl2 (key string)");
+ driver.run("create table testtblpart2 (key string) partitioned by (partkey string)");
+
+ //6 partitions
+ driver.run("alter table default.testtblpart add partition (partkey='a')");
+ driver.run("alter table default.testtblpart add partition (partkey='b')");
+ driver.run("alter table default.testtblpart add partition (partkey='c')");
+ driver.run("alter table testdb1.testtblpart2 add partition (partkey='a')");
+ driver.run("alter table testdb1.testtblpart2 add partition (partkey='b')");
+ driver.run("alter table testdb1.testtblpart2 add partition (partkey='c')");
+
+
+ //create and drop some additional metadata, to test drop counts.
+ driver.run("create database tempdb");
+ driver.run("use tempdb");
+
+ driver.run("create table delete_by_table (key string) partitioned by (partkey string)");
+ driver.run("alter table delete_by_table add partition (partkey='temp')");
+ driver.run("drop table delete_by_table");
+
+ driver.run("create table delete_by_part (key string) partitioned by (partkey string)");
+ driver.run("alter table delete_by_part add partition (partkey='temp')");
+ driver.run("alter table delete_by_part drop partition (partkey='temp')");
+
+ driver.run("create table delete_by_db (key string) partitioned by (partkey string)");
+ driver.run("alter table delete_by_db add partition (partkey='temp')");
+ driver.run("use default");
+ driver.run("drop database tempdb cascade");
+
+ //give timer thread a chance to print the metrics
+ Thread.sleep(2000);
+
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_DATABASES, 1);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_TABLES, 4);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_PARTITIONS, 6);
+
+
+ //to test initial metadata count metrics.
+ hiveConf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, ObjectStore.class.getName());
+ HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("test", hiveConf, false);
+ baseHandler.init();
+ baseHandler.updateMetrics();
+
+ Thread.sleep(2000);
+
+ //1 new db + default
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_DATABASES, 2);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_TABLES, 4);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_PARTITIONS, 6);
+ }
+
@Test
public void testConnections() throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
new file mode 100644
index 0000000..b528376
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.hbase;
+
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.metrics.MetricsTestUtils;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Test HMS Metrics on HBase Metastore
+ */
+public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {
+
+ private static File jsonReportFile;
+
+ @BeforeClass
+ public static void startup() throws Exception {
+ File workDir = new File(System.getProperty("test.tmp.dir"));
+ jsonReportFile = new File(workDir, "json_reporting");
+ jsonReportFile.delete();
+
+ HBaseIntegrationTests.startMiniCluster();
+ }
+
+ @AfterClass
+ public static void shutdown() throws Exception {
+ HBaseIntegrationTests.shutdownMiniCluster();
+ }
+
+ @Before
+ public void before() throws IOException {
+ HBaseReadWrite.setConf(conf);
+ conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
+ "org.apache.hadoop.hive.metastore.hbase.HBaseStore");
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true);
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString());
+ conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
+ SessionState.start(new CliSessionState(conf));
+ driver = new Driver(conf);
+ }
+
+ @Test
+ public void testMetaDataCounts() throws Exception {
+ //1 databases created
+ driver.run("create database testdb1");
+
+ //4 tables
+ driver.run("create table testtbl1 (key string)");
+ driver.run("create table testtblpart (key string) partitioned by (partkey string)");
+ driver.run("use testdb1");
+ driver.run("create table testtbl2 (key string)");
+ driver.run("create table testtblpart2 (key string) partitioned by (partkey string)");
+
+ //6 partitions
+ driver.run("alter table default.testtblpart add partition (partkey='a')");
+ driver.run("alter table default.testtblpart add partition (partkey='b')");
+ driver.run("alter table default.testtblpart add partition (partkey='c')");
+ driver.run("alter table testdb1.testtblpart2 add partition (partkey='a')");
+ driver.run("alter table testdb1.testtblpart2 add partition (partkey='b')");
+ driver.run("alter table testdb1.testtblpart2 add partition (partkey='c')");
+
+
+ //create and drop some additional metadata, to test drop counts.
+ driver.run("create database tempdb");
+ driver.run("use tempdb");
+
+ driver.run("create table delete_by_table (key string) partitioned by (partkey string)");
+ driver.run("alter table delete_by_table add partition (partkey='temp')");
+ driver.run("drop table delete_by_table");
+
+ driver.run("create table delete_by_part (key string) partitioned by (partkey string)");
+ driver.run("alter table delete_by_part add partition (partkey='temp')");
+ driver.run("alter table delete_by_part drop partition (partkey='temp')");
+
+ driver.run("create table delete_by_db (key string) partitioned by (partkey string)");
+ driver.run("alter table delete_by_db add partition (partkey='temp')");
+ driver.run("use default");
+ driver.run("drop database tempdb cascade");
+
+ //give timer thread a chance to print the metrics
+ Thread.sleep(2000);
+
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_DATABASES, 1);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_TABLES, 4);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.COUNTER, MetricsConstant.DELTA_TOTAL_PARTITIONS, 6);
+
+
+ //to test initial metadata count metrics.
+ conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, ObjectStore.class.getName());
+ HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("test", conf, false);
+ baseHandler.init();
+ baseHandler.updateMetrics();
+
+ Thread.sleep(2000);
+
+ //1 new db + default
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_DATABASES, 2);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_TABLES, 4);
+ MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.INIT_TOTAL_PARTITIONS, 6);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
new file mode 100644
index 0000000..fa5922f
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.metrics.common.Metrics;
+import org.apache.hadoop.hive.common.metrics.common.MetricsConstant;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Report metrics of metadata added, deleted by this Hive Metastore.
+ */
+public class HMSMetricsListener extends MetaStoreEventListener {
+
+ public static final Logger LOGGER = LoggerFactory.getLogger(HMSMetricsListener.class);
+ private Metrics metrics;
+
+ public HMSMetricsListener(Configuration config, Metrics metrics) {
+ super(config);
+ this.metrics = metrics;
+ }
+
+ @Override
+ public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException {
+ if (metrics != null) {
+ try {
+ metrics.incrementCounter(MetricsConstant.DELTA_TOTAL_DATABASES);
+ } catch (IOException e) {
+ LOGGER.warn("Error updating metadata metrics", e);
+ }
+ }
+ }
+
+ @Override
+ public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException {
+ if (metrics != null) {
+ try {
+ metrics.decrementCounter(MetricsConstant.DELTA_TOTAL_DATABASES);
+ } catch (IOException e) {
+ LOGGER.warn("Error updating metadata metrics", e);
+ }
+ }
+ }
+
+ @Override
+ public void onCreateTable(CreateTableEvent tableEvent) throws MetaException {
+ if (metrics != null) {
+ try {
+ metrics.incrementCounter(MetricsConstant.DELTA_TOTAL_TABLES);
+ } catch (IOException e) {
+ LOGGER.warn("Error updating metadata metrics", e);
+ }
+ }
+ }
+
+ @Override
+ public void onDropTable(DropTableEvent tableEvent) throws MetaException {
+ if (metrics != null) {
+ try {
+ metrics.decrementCounter(MetricsConstant.DELTA_TOTAL_TABLES);
+ } catch (IOException e) {
+ LOGGER.warn("Error updating metadata metrics", e);
+ }
+ }
+ }
+
+ @Override
+ public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException {
+ if (metrics != null) {
+ try {
+ metrics.decrementCounter(MetricsConstant.DELTA_TOTAL_PARTITIONS);
+ } catch (IOException e) {
+ LOGGER.warn("Error updating metadata metrics", e);
+ }
+ }
+ }
+
+ @Override
+ public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException {
+ if (metrics != null) {
+ try {
+ metrics.incrementCounter(MetricsConstant.DELTA_TOTAL_PARTITIONS);
+ } catch (IOException e) {
+ LOGGER.warn("Error updating metadata metrics", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 00602e1..fec8ea0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -28,6 +28,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Multimaps;
import org.apache.commons.cli.OptionBuilder;
+import org.apache.hadoop.hive.common.metrics.common.MetricsVariable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -203,6 +204,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
private static String currentUrl;
+ //For Metrics
+ private int initDatabaseCount, initTableCount, initPartCount;
+
private Warehouse wh; // hdfs warehouse
private static final ThreadLocal<RawStore> threadLocalMS =
new ThreadLocal<RawStore>() {
@@ -385,12 +389,43 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
}
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ LOG.info("Begin calculating metadata count metrics.");
+ updateMetrics();
+ LOG.info("Finished metadata count metrics: " + initDatabaseCount + " databases, " + initTableCount +
+ " tables, " + initPartCount + " partitions.");
+ metrics.addGauge(MetricsConstant.INIT_TOTAL_DATABASES, new MetricsVariable() {
+ @Override
+ public Object getValue() {
+ return initDatabaseCount;
+ }
+ });
+ metrics.addGauge(MetricsConstant.INIT_TOTAL_TABLES, new MetricsVariable() {
+ @Override
+ public Object getValue() {
+ return initTableCount;
+ }
+ });
+ metrics.addGauge(MetricsConstant.INIT_TOTAL_PARTITIONS, new MetricsVariable() {
+ @Override
+ public Object getValue() {
+ return initPartCount;
+ }
+ });
+ }
+
preListeners = MetaStoreUtils.getMetaStoreListeners(MetaStorePreEventListener.class,
hiveConf,
hiveConf.getVar(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS));
listeners = MetaStoreUtils.getMetaStoreListeners(MetaStoreEventListener.class, hiveConf,
hiveConf.getVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS));
listeners.add(new SessionPropertiesListener(hiveConf));
+
+ if (metrics != null) {
+ listeners.add(new HMSMetricsListener(hiveConf, metrics));
+ }
+
endFunctionListeners = MetaStoreUtils.getMetaStoreListeners(
MetaStoreEndFunctionListener.class, hiveConf,
hiveConf.getVar(HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS));
@@ -1633,6 +1668,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
partNames.add(Warehouse.makePartName(tbl.getPartitionKeys(), part.getValues()));
}
+ for (MetaStoreEventListener listener : listeners) {
+ //No drop part listener events fired for public listeners historically, for drop table case.
+ //Limiting to internal listeners for now, to avoid unexpected calls for public listeners.
+ if (listener instanceof HMSMetricsListener) {
+ for (Partition part : partsToDelete) {
+ listener.onDropPartition(null);
+ }
+ }
+ }
ms.dropPartitions(dbName, tableName, partNames);
}
@@ -5730,6 +5774,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
getMS().putFileMetadata(req.getFileIds(), null);
return new ClearFileMetadataResult();
}
+
+ @VisibleForTesting
+ public void updateMetrics() throws MetaException {
+ initTableCount = getMS().getTableCount();
+ initPartCount = getMS().getPartitionCount();
+ initDatabaseCount = getMS().getDatabaseCount();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 1c0ab6d..abfe2b8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -37,8 +37,11 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
@@ -213,7 +216,6 @@ public class ObjectStore implements RawStore, Configurable {
private volatile int openTrasactionCalls = 0;
private Transaction currentTransaction = null;
private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE;
-
private Pattern partitionValidationPattern;
/**
@@ -1049,6 +1051,40 @@ public class ObjectStore implements RawStore, Configurable {
return tbls;
}
+ public int getDatabaseCount() throws MetaException {
+ return getObjectCount("name", MDatabase.class.getName());
+ }
+
+ public int getPartitionCount() throws MetaException {
+ return getObjectCount("partitionName", MPartition.class.getName());
+ }
+
+ public int getTableCount() throws MetaException {
+ return getObjectCount("tableName", MTable.class.getName());
+ }
+
+ private int getObjectCount(String fieldName, String objName) {
+ Long result = 0L;
+ boolean commited = false;
+ Query query = null;
+ try {
+ openTransaction();
+ String queryStr =
+ "select count(" + fieldName + ") from " + objName;
+ query = pm.newQuery(queryStr);
+ result = (Long) query.execute();
+ commited = commitTransaction();
+ } finally {
+ if (!commited) {
+ rollbackTransaction();
+ }
+ if (query != null) {
+ query.closeAll();
+ }
+ }
+ return result.intValue();
+ }
+
@Override
public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes)
throws MetaException {
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
index 5b36b03..e118a3b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
@@ -633,4 +634,22 @@ public interface RawStore extends Configurable {
void getFileMetadataByExpr(List<Long> fileIds, FileMetadataExprType type, byte[] expr,
ByteBuffer[] metadatas, ByteBuffer[] exprResults, boolean[] eliminated)
throws MetaException;
+
+ /**
+ * Gets total number of tables.
+ */
+ @InterfaceStability.Evolving
+ int getTableCount() throws MetaException;
+
+ /**
+ * Gets total number of partitions.
+ */
+ @InterfaceStability.Evolving
+ int getPartitionCount() throws MetaException;
+
+ /**
+ * Gets total number of databases.
+ */
+ @InterfaceStability.Evolving
+ int getDatabaseCount() throws MetaException;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
index 2fb3e8f..287394e 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
@@ -19,8 +19,10 @@
package org.apache.hadoop.hive.metastore.hbase;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterators;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -421,6 +423,12 @@ public class HBaseReadWrite {
}
}
+ int getDatabaseCount() throws IOException {
+ Filter fil = new FirstKeyOnlyFilter();
+ Iterator<Result> iter = scan(DB_TABLE, fil);
+ return Iterators.size(iter);
+ }
+
/**********************************************************************************************
* Function related methods
*********************************************************************************************/
@@ -865,6 +873,12 @@ public class HBaseReadWrite {
return lines;
}
+ int getPartitionCount() throws IOException {
+ Filter fil = new FirstKeyOnlyFilter();
+ Iterator<Result> iter = scan(PART_TABLE, fil);
+ return Iterators.size(iter);
+ }
+
private String printOnePartition(Result result) throws IOException, TException {
byte[] key = result.getRow();
HBaseUtils.StorageDescriptorParts sdParts =
@@ -1671,6 +1685,12 @@ public class HBaseReadWrite {
return lines;
}
+ int getTableCount() throws IOException {
+ Filter fil = new FirstKeyOnlyFilter();
+ Iterator<Result> iter = scan(TABLE_TABLE, fil);
+ return Iterators.size(iter);
+ }
+
private String printOneTable(Result result) throws IOException, TException {
byte[] key = result.getRow();
HBaseUtils.StorageDescriptorParts sdParts =
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
index 98e6c75..b9509ab 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
@@ -240,6 +240,16 @@ public class HBaseStore implements RawStore {
}
@Override
+ public int getDatabaseCount() throws MetaException {
+ try {
+ return getHBase().getDatabaseCount();
+ } catch (IOException e) {
+ LOG.error("Unable to get database count", e);
+ throw new MetaException("Error scanning databases");
+ }
+ }
+
+ @Override
public boolean createType(Type type) {
throw new UnsupportedOperationException();
}
@@ -561,6 +571,16 @@ public class HBaseStore implements RawStore {
}
@Override
+ public int getTableCount() throws MetaException {
+ try {
+ return getHBase().getTableCount();
+ } catch (IOException e) {
+ LOG.error("Unable to get table count", e);
+ throw new MetaException("Error scanning tables");
+ }
+ }
+
+ @Override
public List<String> listTableNamesByFilter(String dbName, String filter, short max_tables) throws
MetaException, UnknownDBException {
// TODO needs to wait until we support pushing filters into HBase.
@@ -883,6 +903,16 @@ public class HBaseStore implements RawStore {
*/
@Override
+ public int getPartitionCount() throws MetaException {
+ try {
+ return getHBase().getPartitionCount();
+ } catch (IOException e) {
+ LOG.error("Unable to get partition count", e);
+ throw new MetaException("Error scanning partitions");
+ }
+ }
+
+ @Override
public boolean addRole(String roleName, String ownerName) throws InvalidObjectException,
MetaException, NoSuchObjectException {
int now = (int)(System.currentTimeMillis()/1000);
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index 9a1d159..c1156b3 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -788,4 +788,19 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
public void getFileMetadataByExpr(List<Long> fileIds, FileMetadataExprType type, byte[] expr,
ByteBuffer[] metadatas, ByteBuffer[] stripeBitsets, boolean[] eliminated) {
}
+
+ @Override
+ public int getTableCount() throws MetaException {
+ return objectStore.getTableCount();
+ }
+
+ @Override
+ public int getPartitionCount() throws MetaException {
+ return objectStore.getPartitionCount();
+ }
+
+ @Override
+ public int getDatabaseCount() throws MetaException {
+ return objectStore.getDatabaseCount();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/546c75c5/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 8dde0af..bf20e99 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -804,6 +804,21 @@ public class DummyRawStoreForJdoConnection implements RawStore {
public void getFileMetadataByExpr(List<Long> fileIds, FileMetadataExprType type, byte[] expr,
ByteBuffer[] metadatas, ByteBuffer[] stripeBitsets, boolean[] eliminated) {
}
+
+ @Override
+ public int getTableCount() throws MetaException {
+ return 0;
+ }
+
+ @Override
+ public int getPartitionCount() throws MetaException {
+ return 0;
+ }
+
+ @Override
+ public int getDatabaseCount() throws MetaException {
+ return 0;
+ }
}