You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/06/26 22:54:08 UTC
[2/4] hbase git commit: HBASE-19722 Meta query statistics metrics
source
HBASE-19722 Meta query statistics metrics source
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7ad85fb2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7ad85fb2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7ad85fb2
Branch: refs/heads/branch-1.4
Commit: 7ad85fb29ab469e9a2aa572f6e9c630272341e91
Parents: 88c280e
Author: Xu Cang <xc...@salesforce.com>
Authored: Mon May 14 14:24:51 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Jun 25 18:27:59 2018 -0700
----------------------------------------------------------------------
.../hbase/coprocessor/MetaTableMetrics.java | 302 +++++++++++++++++++
.../hbase/coprocessor/TestMetaTableMetrics.java | 225 ++++++++++++++
2 files changed, 527 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7ad85fb2/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetaTableMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetaTableMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetaTableMetrics.java
new file mode 100644
index 0000000..6945ffa
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetaTableMetrics.java
@@ -0,0 +1,302 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.metrics.Meter;
+import org.apache.hadoop.hbase.metrics.Metric;
+import org.apache.hadoop.hbase.metrics.MetricRegistry;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
+/**
+ * A coprocessor that collects metrics from meta table.
+ * <p>
+ * These metrics will be available through the regular Hadoop metrics2 sinks (ganglia, opentsdb,
+ * etc) as well as JMX output.
+ * </p>
+ *
+ * @see MetaTableMetrics
+ */
+
+@InterfaceAudience.Private
+public class MetaTableMetrics extends BaseRegionObserver {
+
+ private Map<String, Optional<Metric>> requestsMap;
+ private RegionCoprocessorEnvironment regionCoprocessorEnv;
+ private boolean active = false;
+ private ImmutableMap<Class, MetaTableOps> opsNameMap =
+ ImmutableMap.<Class, MetaTableOps>builder().put(Put.class, MetaTableOps.PUT)
+ .put(Get.class, MetaTableOps.GET).put(Delete.class, MetaTableOps.DELETE).build();
+
+ @Override
+ public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
+ throws IOException {
+ if (!active || !isMetaTableOp(e)) {
+ return;
+ }
+ super.preGetOp(e, get, results);
+ tableMetricRegisterAndMark(e, get);
+ clientMetricRegisterAndMark(e);
+ regionMetricRegisterAndMark(e, get);
+ opMetricRegisterAndMark(e, get);
+ opWithClientMetricRegisterAndMark(e, get);
+ }
+
+ @Override
+ public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit,
+ Durability durability) throws IOException {
+ if (!active || !isMetaTableOp(e)) {
+ return;
+ }
+ super.prePut(e, put, edit, durability);
+ tableMetricRegisterAndMark(e, put);
+ clientMetricRegisterAndMark(e);
+ regionMetricRegisterAndMark(e, put);
+ opMetricRegisterAndMark(e, put);
+ opWithClientMetricRegisterAndMark(e, put);
+ }
+
+ @Override
+ public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete,
+ WALEdit edit, Durability durability) {
+ if (!active || !isMetaTableOp(e)) {
+ return;
+ }
+ tableMetricRegisterAndMark(e, delete);
+ clientMetricRegisterAndMark(e);
+ regionMetricRegisterAndMark(e, delete);
+ opMetricRegisterAndMark(e, delete);
+ opWithClientMetricRegisterAndMark(e, delete);
+ }
+
+ private void markMeterIfPresent(String requestMeter) {
+ if (requestMeter.isEmpty()) {
+ return;
+ }
+ Metric metric =
+ requestsMap.get(requestMeter).isPresent() ? requestsMap.get(requestMeter).get() : null;
+ if (metric != null) {
+ ((Meter) metric).mark();
+ }
+ }
+
+ private void registerMeterIfNotPresent(ObserverContext<RegionCoprocessorEnvironment> e,
+ String requestMeter) {
+ if (requestMeter.isEmpty()) {
+ return;
+ }
+ if (!requestsMap.containsKey(requestMeter)) {
+ MetricRegistry registry = regionCoprocessorEnv.getMetricRegistryForRegionServer();
+ registry.meter(requestMeter);
+ requestsMap.put(requestMeter, registry.get(requestMeter));
+ }
+ }
+
+ /**
+ * Get table name from Ops such as: get, put, delete.
+ *
+ * @param op Op such as get, put or delete.
+ */
+ private String getTableNameFromOp(Object op) {
+ String tableName = null;
+ if (op instanceof Mutation) {
+ String tableRowKey = new String(((Mutation) op).getRow());
+ if (tableRowKey.isEmpty()) {
+ return null;
+ }
+ tableName = tableRowKey.split(",").length > 0 ? tableRowKey.split(",")[0] : null;
+ } else if (op instanceof Row) {
+ String tableRowKey = new String(((Row) op).getRow());
+ if (tableRowKey.isEmpty()) {
+ return null;
+ }
+ tableName = tableRowKey.split(",").length > 0 ? tableRowKey.split(",")[0] : null;
+ }
+ return tableName;
+ }
+
+ /**
+ * Get regionId from Ops such as: get, put, delete.
+ *
+ * @param op Op such as get, put or delete.
+ */
+ private String getReginoIdFromOp(Object op) {
+ String regionId = null;
+ if (op instanceof Mutation) {
+ String tableRowKey = new String(((Mutation) op).getRow());
+ if (tableRowKey.isEmpty()) {
+ return null;
+ }
+ regionId = tableRowKey.split(",").length > 2 ? tableRowKey.split(",")[2] : null;
+ } else if (op instanceof Row) {
+ String tableRowKey = new String(((Row) op).getRow());
+ if (tableRowKey.isEmpty()) {
+ return null;
+ }
+ regionId = tableRowKey.split(",").length > 2 ? tableRowKey.split(",")[2] : null;
+ }
+ return regionId;
+ }
+
+ private boolean isMetaTableOp(ObserverContext<RegionCoprocessorEnvironment> e) {
+ return TableName.META_TABLE_NAME.toString()
+ .equals(new String(e.getEnvironment().getRegionInfo().getTable().getName()));
+ }
+
+ private void clientMetricRegisterAndMark(ObserverContext<RegionCoprocessorEnvironment> e) {
+ String clientIP = RpcServer.getRemoteIp() != null ? RpcServer.getRemoteIp().toString() : "";
+ String clientRequestMeter = clientRequestMeterName(clientIP);
+ registerMeterIfNotPresent(e, clientRequestMeter);
+ markMeterIfPresent(clientRequestMeter);
+ }
+
+ private void tableMetricRegisterAndMark(ObserverContext<RegionCoprocessorEnvironment> e,
+ Object op) {
+ if (!(op instanceof Mutation) && !(op instanceof Row)) {
+ return;
+ }
+ // Mark the meta table meter whenever the coprocessor is called
+ String tableName = getTableNameFromOp(op);
+ String tableRequestMeter = tableMeterName(tableName);
+ registerMeterIfNotPresent(e, tableRequestMeter);
+ markMeterIfPresent(tableRequestMeter);
+ }
+
+ private void regionMetricRegisterAndMark(ObserverContext<RegionCoprocessorEnvironment> e,
+ Object op) {
+ if (!(op instanceof Mutation) && !(op instanceof Row)) {
+ return;
+ }
+ // Mark the meta table meter whenever the coprocessor is called
+ String regionId = getReginoIdFromOp(op);
+ String regionRequestMeter = regionMeterName(regionId);
+ registerMeterIfNotPresent(e, regionRequestMeter);
+ markMeterIfPresent(regionRequestMeter);
+ }
+
+ private void opMetricRegisterAndMark(ObserverContext<RegionCoprocessorEnvironment> e, Object op) {
+ String opMeterName = opMeterName(op);
+ registerMeterIfNotPresent(e, opMeterName);
+ markMeterIfPresent(opMeterName);
+ }
+
+ private void opWithClientMetricRegisterAndMark(ObserverContext<RegionCoprocessorEnvironment> e,
+ Object op) {
+ String opWithClientMeterName = opWithClientMeterName(op);
+ registerMeterIfNotPresent(e, opWithClientMeterName);
+ markMeterIfPresent(opWithClientMeterName);
+ }
+
+ private String opWithClientMeterName(Object op) {
+ String clientIP = RpcServer.getRemoteIp() != null ? RpcServer.getRemoteIp().toString() : "";
+ if (clientIP.isEmpty()) {
+ return "";
+ }
+ MetaTableOps ops = opsNameMap.get(op.getClass());
+ String opWithClientMeterName = "";
+ switch (ops) {
+ case GET:
+ opWithClientMeterName = String.format("MetaTable_client_%s_get_request", clientIP);
+ break;
+ case PUT:
+ opWithClientMeterName = String.format("MetaTable_client_%s_put_request", clientIP);
+ break;
+ case DELETE:
+ opWithClientMeterName = String.format("MetaTable_client_%s_delete_request", clientIP);
+ break;
+ default:
+ break;
+ }
+ return opWithClientMeterName;
+ }
+
+ private String opMeterName(Object op) {
+ MetaTableOps ops = opsNameMap.get(op.getClass());
+ String opMeterName = "";
+ switch (ops) {
+ case GET:
+ opMeterName = "MetaTable_get_request";
+ break;
+ case PUT:
+ opMeterName = "MetaTable_put_request";
+ break;
+ case DELETE:
+ opMeterName = "MetaTable_delete_request";
+ break;
+ default:
+ break;
+ }
+ return opMeterName;
+ }
+
+ private String tableMeterName(String tableName) {
+ return String.format("MetaTable_table_%s_request", tableName);
+ }
+
+ private String clientRequestMeterName(String clientIP) {
+ if (clientIP.isEmpty()) {
+ return "";
+ }
+ return String.format("MetaTable_client_%s_request", clientIP);
+ }
+
+ private String regionMeterName(String regionId) {
+ return String.format("MetaTable_region_%s_request", regionId);
+ }
+
+ @Override
+ public void start(CoprocessorEnvironment env) throws IOException {
+ super.start(env);
+ if (env instanceof RegionCoprocessorEnvironment
+ && ((RegionCoprocessorEnvironment) env).getRegionInfo().getTable() != null
+ && ((RegionCoprocessorEnvironment) env).getRegionInfo().getTable().getName() != null
+ && new String(((RegionCoprocessorEnvironment) env).getRegionInfo().getTable().getName())
+ .equals(TableName.META_TABLE_NAME.toString())) {
+ regionCoprocessorEnv = (RegionCoprocessorEnvironment) env;
+ requestsMap = new ConcurrentHashMap<>();
+ // only be active mode when this region holds meta table.
+ active = true;
+ }
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment e) throws IOException {
+ // since meta region can move around, clear stale metrics when stop.
+ for (String meterName : requestsMap.keySet()) {
+ MetricRegistry registry = regionCoprocessorEnv.getMetricRegistryForRegionServer();
+ registry.remove(meterName);
+ }
+ super.stop(e);
+ }
+
+ enum MetaTableOps {
+ GET, PUT, DELETE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7ad85fb2/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMetaTableMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMetaTableMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMetaTableMetrics.java
new file mode 100644
index 0000000..5b90d0c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMetaTableMetrics.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.coprocessor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanInfo;
+import javax.management.MBeanServerConnection;
+import javax.management.ObjectInstance;
+import javax.management.ObjectName;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.JMXListener;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+@Category({ CoprocessorTests.class, MediumTests.class })
+public class TestMetaTableMetrics {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestMetaTableMetrics.class);
+
+ private static HBaseTestingUtility UTIL = null;
+ private static final TableName NAME1 = TableName.valueOf("TestExampleMetaTableMetricsOne");
+ private static final byte[] FAMILY = Bytes.toBytes("f");
+ private static final byte[] QUALIFIER = Bytes.toBytes("q");
+ private static final int NUM_ROWS = 5;
+ private static final String value = "foo";
+ private static Configuration conf = null;
+ private static int connectorPort = 61120;
+
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ UTIL = new HBaseTestingUtility();
+ conf = UTIL.getConfiguration();
+ conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
+ Random rand = new Random();
+ for (int i = 0; i < 10; i++) {
+ do {
+ int sign = i % 2 == 0 ? 1 : -1;
+ connectorPort += sign * rand.nextInt(100);
+ } while (!HBaseTestingUtility.available(connectorPort));
+ try {
+ conf.setInt("regionserver.rmi.registry.port", connectorPort);
+ UTIL.startMiniCluster();
+ UTIL.createTable(NAME1, new byte[][]{FAMILY});
+ LOG.error("util to string" + UTIL.toString());
+ break;
+ } catch (Exception e) {
+ LOG.debug("Encountered exception when starting mini cluster. Trying port " + connectorPort,
+ e);
+ try {
+ // this is to avoid "IllegalStateException: A mini-cluster is already running"
+ UTIL.shutdownMiniCluster();
+ } catch (Exception ex) {
+ LOG.debug("Encountered exception shutting down cluster", ex);
+ }
+ }
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ private void writeData(Table t) throws IOException {
+ List<Put> puts = new ArrayList<>(NUM_ROWS);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ Put p = new Put(Bytes.toBytes(i + 1));
+ p.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(value));
+ puts.add(p);
+ }
+ t.put(puts);
+ }
+
+ private Set<String> readJmxMetricsWithRetry() throws IOException {
+ final int count = 0;
+ for (int i = 0; i < 10; i++) {
+ Set<String> metrics = readJmxMetrics();
+ if (metrics != null) {
+ return metrics;
+ }
+ LOG.warn("Failed to get jmxmetrics... sleeping, retrying; " + i + " of " + count + " times");
+ Threads.sleep(1000);
+ }
+ return null;
+ }
+
+ /**
+ * Read the attributes from Hadoop->HBase->RegionServer->MetaTableMetrics in JMX
+ * @throws IOException when fails to retrieve jmx metrics.
+ */
+ // this method comes from this class: TestStochasticBalancerJmxMetrics with minor modifications.
+ private Set<String> readJmxMetrics() throws IOException {
+ JMXConnector connector = null;
+ ObjectName target = null;
+ MBeanServerConnection mb = null;
+ try {
+ connector =
+ JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort, connectorPort));
+ mb = connector.getMBeanServerConnection();
+
+ @SuppressWarnings("JdkObsolete")
+ Hashtable<String, String> pairs = new Hashtable<>();
+ pairs.put("service", "HBase");
+ pairs.put("name", "RegionServer");
+ pairs.put("sub",
+ "Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor"
+ + ".MetaTableMetrics");
+ target = new ObjectName("Hadoop", pairs);
+ MBeanInfo beanInfo = mb.getMBeanInfo(target);
+
+ Set<String> existingAttrs = new HashSet<>();
+ for (MBeanAttributeInfo attrInfo : beanInfo.getAttributes()) {
+ existingAttrs.add(attrInfo.getName());
+ }
+ return existingAttrs;
+ } catch (Exception e) {
+ LOG.warn("Failed to get bean." + target, e);
+ if (mb != null) {
+ Set<ObjectInstance> instances = mb.queryMBeans(null, null);
+ Iterator<ObjectInstance> iterator = instances.iterator();
+ LOG.warn("MBean Found:");
+ while (iterator.hasNext()) {
+ ObjectInstance instance = iterator.next();
+ LOG.warn("Class Name: " + instance.getClassName());
+ LOG.warn("Object Name: " + instance.getObjectName());
+ }
+ }
+ } finally {
+ if (connector != null) {
+ try {
+ connector.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return null;
+ }
+
+ // verifies meta table metrics exist from jmx
+ // for one table, there should be 5 MetaTable_table_<TableName> metrics.
+ // such as:
+ // [Time-limited test] example.TestMetaTableMetrics(204): ==
+ // MetaTable_table_TestExampleMetaTableMetricsOne_request_count
+ // [Time-limited test] example.TestMetaTableMetrics(204): ==
+ // MetaTable_table_TestExampleMetaTableMetricsOne_request_mean_rate
+ // [Time-limited test] example.TestMetaTableMetrics(204): ==
+ // MetaTable_table_TestExampleMetaTableMetricsOne_request_1min_rate
+ // [Time-limited test] example.TestMetaTableMetrics(204): ==
+ // MetaTable_table_TestExampleMetaTableMetricsOne_request_5min_rate
+ // [Time-limited test] example.TestMetaTableMetrics(204): ==
+ // MetaTable_table_TestExampleMetaTableMetricsOne_request_15min_rate
+ @Test
+ public void test() throws IOException, InterruptedException {
+ try (Table t = UTIL.getConnection().getTable(NAME1)) {
+ writeData(t);
+ // Flush the data
+ UTIL.flush(NAME1);
+ // Issue a compaction
+ UTIL.compact(NAME1, true);
+ Thread.sleep(2000);
+ }
+ Set<String> jmxMetrics = readJmxMetricsWithRetry();
+ assertNotNull(jmxMetrics);
+ long name1TableMetricsCount = 0;
+ for(String metric : jmxMetrics) {
+ if (metric.contains("MetaTable_table_" + NAME1)){
+ name1TableMetricsCount++;
+ }
+ }
+ assertEquals(5L, name1TableMetricsCount);
+
+ String putWithClientMetricNameRegex = "MetaTable_client_.+_put_request.*";
+ long putWithClientMetricsCount = 0;
+ for(String metric : jmxMetrics) {
+ if(metric.matches(putWithClientMetricNameRegex)) {
+ putWithClientMetricsCount++;
+ }
+ }
+ assertEquals(5L, putWithClientMetricsCount);
+
+
+
+
+ }
+
+}