You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/05/07 20:31:11 UTC
svn commit: r1480004 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/ipc/
main/java/org/apache/hadoop/hbase/regionserver/metrics/
main/java/org/apache/hadoop/hbase/util/
test/java/org/apache/hadoop/hbase/regionserver/metrics/
Author: liyin
Date: Tue May 7 18:31:06 2013
New Revision: 1480004
URL: http://svn.apache.org/r1480004
Log:
[HBASE-6404] Collect p95 stats for get/put/delete/next
Author: gauravm
Summary: Collect p95 stats for get / put / delete / next operations by having a histogram and a percentile metric (with percentile = 0.95) for each operation, respectively. The method of stat collections is similar to D715751.
Test Plan: Unit tests and checked on Titan Shadow.
Reviewers: liyintang, adela, manukranthk
Reviewed By: manukranthk
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D779670
Task ID: 1200149
Added:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RpcMetricWrapper.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRpcMetricWrapper.java
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/PercentileMetric.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Histogram.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java?rev=1480004&r1=1480003&r2=1480004&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java Tue May 7 18:31:06 2013
@@ -22,12 +22,14 @@ package org.apache.hadoop.hbase.ipc;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.regionserver.metrics.RpcMetricWrapper;
import org.apache.hadoop.metrics.MetricsContext;
import org.apache.hadoop.metrics.MetricsRecord;
import org.apache.hadoop.metrics.MetricsUtil;
import org.apache.hadoop.metrics.Updater;
import org.apache.hadoop.metrics.util.MetricsRegistry;
import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
+import java.util.concurrent.ConcurrentHashMap;
/**
*
@@ -60,7 +62,16 @@ public class HBaseRpcMetrics implements
rpcStatistics = new HBaseRPCStatistics(this.registry, hostName, port);
}
-
+ /**
+ * A metric name to RPCMetric object map. We do not keep RpcMetricWrapper objects
+ * in the registry, because firstly the RpcMetricWrapper class does not extend
+ * MetricsBase (since it is a container class, and not an atomic metric),
+ * and secondly we do not want to confuse between the RpcMetricWrapper name and the
+ * name of the rate metric contained within a RpcMetricWrapper object, when
+ * querying the registry.
+ */
+ private final ConcurrentHashMap<String, RpcMetricWrapper> rpcMetricWrapperMap =
+ new ConcurrentHashMap<String, RpcMetricWrapper>();
/**
* The metrics variables are public:
* - they can be set directly by calling their set/inc methods
@@ -71,18 +82,25 @@ public class HBaseRpcMetrics implements
public MetricsTimeVaryingRate rpcQueueTime = new MetricsTimeVaryingRate("RpcQueueTime", registry);
public MetricsTimeVaryingRate rpcProcessingTime = new MetricsTimeVaryingRate("RpcProcessingTime", registry);
- //public Map <String, MetricsTimeVaryingRate> metricsList = Collections.synchronizedMap(new HashMap<String, MetricsTimeVaryingRate>());
-
-
- private MetricsTimeVaryingRate get(String key) {
- return (MetricsTimeVaryingRate) registry.get(key);
+ private RpcMetricWrapper get(String name) {
+ return rpcMetricWrapperMap.get(name);
}
- private MetricsTimeVaryingRate create(String key) {
- return new MetricsTimeVaryingRate(key, this.registry);
+
+ private RpcMetricWrapper create(String name) {
+ RpcMetricWrapper m = new RpcMetricWrapper(name, this.registry);
+ rpcMetricWrapperMap.put(name, m);
+ return m;
}
+ /**
+ * Increment a metric. This would increment two different metrics.
+ * The first one being the standard MetricsTimeVaryingRate, and the
+ * other will be a PercentMetric.
+ * @param name
+ * @param amt
+ */
public void inc(String name, int amt) {
- MetricsTimeVaryingRate m = get(name);
+ RpcMetricWrapper m = get(name);
if (m == null) {
synchronized (this) {
if ((m = get(name)) == null) {
@@ -104,13 +122,13 @@ public class HBaseRpcMetrics implements
rpcProcessingTime.pushMetric(metricsRecord);
rpcProcessingTime.resetMinMax();
- synchronized (registry) {
- // Iterate through the registry to propagate the different rpc metrics.
+ synchronized (rpcMetricWrapperMap) {
+ // Iterate through the rpcMetricWrapperMap to propagate the different rpc metrics.
- for (String metricName : registry.getKeyList() ) {
- MetricsTimeVaryingRate value = (MetricsTimeVaryingRate) registry.get(metricName);
+ for (String metricName : rpcMetricWrapperMap.keySet()) {
+ RpcMetricWrapper value = get(metricName);
value.pushMetric(metricsRecord);
- value.resetMinMax();
+ value.resetMetric();
}
}
metricsRecord.update();
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/PercentileMetric.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/PercentileMetric.java?rev=1480004&r1=1480003&r2=1480004&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/PercentileMetric.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/PercentileMetric.java Tue May 7 18:31:06 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.regionse
import org.apache.hadoop.hbase.util.Histogram;
import org.apache.hadoop.metrics.util.MetricsLongValue;
import org.apache.hadoop.metrics.util.MetricsRegistry;
+import org.apache.hadoop.metrics.MetricsRecord;
/**
* Used the org.apache.hadoop.hbase.util.Histogram to maintain time varying
@@ -26,13 +27,14 @@ import org.apache.hadoop.metrics.util.Me
* a stream of data supplied to the PercentileMetric without actually
* storing the data.
*/
-public class PercentileMetric extends MetricsLongValue{
+public class PercentileMetric extends MetricsLongValue {
public static final int HISTOGRAM_NUM_BUCKETS_DEFAULT = 20;
public static final double HISTOGRAM_MINVALUE_DEFAULT = 0.0;
public static final double HISTOGRAM_MAXVALUE_DEFAULT = 1000000000.0;
public static final double DEFAULT_PERCENTILE = 99.0;
public static final long DEFAULT_SAMPLE_WINDOW = 60;
public static final double P99 = 99.0;
+ public static final double P95 = 95.0;
private int numBuckets;
private double percentile;
@@ -91,4 +93,21 @@ public class PercentileMetric extends Me
public void refresh() {
underlyingHistogram.refresh(this.numBuckets);
}
+
+ /**
+ * Add a value in the underlying histogram.
+ * @param value The value to be added.
+ */
+ public void addValueInHistogram(long value) {
+ underlyingHistogram.addValue(value);
+ }
+
+ /**
+ * Push the metric value to the <code>MetricsRecord</code> object
+ * @param mr
+ */
+ public void pushMetric(final MetricsRecord mr) {
+ this.updateMetric();
+ mr.setMetric(getName(), (long)getValue());
+ }
}
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RpcMetricWrapper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RpcMetricWrapper.java?rev=1480004&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RpcMetricWrapper.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RpcMetricWrapper.java Tue May 7 18:31:06 2013
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.metrics;
+
+import org.apache.hadoop.hbase.util.Histogram;
+import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+import org.apache.hadoop.metrics.MetricsRecord;
+
+/**
+ * This is a container class for metrics associated with RPC calls. It keeps a
+ * rate metric, which provides the min/max/avg for the metric. It also contains
+ * a percentile metric, which provides the approx value at a specific
+ * percentile (in this case, 95.0), in the distribution of the metric.
+ */
+public class RpcMetricWrapper {
+ public static final double percentile = PercentileMetric.P95;
+ private PercentileMetric percentileMetric;
+ private MetricsTimeVaryingRate rateMetric;
+
+ /**
+ * @param key The key for RpcMetricWrapper
+ * @param metricsRegistry The appropriate metric registry
+ */
+ public RpcMetricWrapper(String key, MetricsRegistry metricsRegistry) {
+ this.percentileMetric =
+ new PercentileMetric((key + "_p" + (long)percentile),
+ metricsRegistry, new Histogram(), percentile,
+ PercentileMetric.HISTOGRAM_NUM_BUCKETS_DEFAULT);
+ this.rateMetric = new MetricsTimeVaryingRate(key, metricsRegistry);
+ }
+
+ /**
+ * Increment the metric by <code>amt</code>.
+ * @param amt
+ */
+ public void inc(int amt) {
+ rateMetric.inc(amt);
+ percentileMetric.addValueInHistogram(amt);
+ }
+
+ /**
+ * Push the metric to the MetricsRecord.
+ * @param metricsRecord
+ */
+ public void pushMetric(MetricsRecord metricsRecord) {
+ percentileMetric.pushMetric(metricsRecord);
+ rateMetric.pushMetric(metricsRecord);
+ }
+
+ /**
+ * Reset the metric
+ */
+ public void resetMetric() {
+ percentileMetric.refresh();
+ rateMetric.resetMinMax();
+ }
+}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Histogram.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Histogram.java?rev=1480004&r1=1480003&r2=1480004&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Histogram.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Histogram.java Tue May 7 18:31:06 2013
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.util;
+import org.apache.hadoop.hbase.regionserver.metrics.PercentileMetric;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -51,6 +52,16 @@ public class Histogram {
final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
public static final Log LOG = LogFactory.getLog(Histogram.class.getName());
+ /**
+ * Create a histogram with the default values of number of buckets,
+ * and min/max for the values.
+ */
+ public Histogram() {
+ this(PercentileMetric.HISTOGRAM_NUM_BUCKETS_DEFAULT,
+ PercentileMetric.HISTOGRAM_MINVALUE_DEFAULT,
+ PercentileMetric.HISTOGRAM_MAXVALUE_DEFAULT);
+ }
+
// Bucket indexing is from 1 to N
public Histogram(int numBuckets, Double minValue, Double maxValue) {
if (numBuckets < 1 || minValue >= maxValue) {
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRpcMetricWrapper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRpcMetricWrapper.java?rev=1480004&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRpcMetricWrapper.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRpcMetricWrapper.java Tue May 7 18:31:06 2013
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.metrics;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.util.MetricsRegistry;
+
+import java.util.HashMap;
+
+/**
+ * A dummy MetricsRecord class which essentially just maintains a map of
+ * metric name to value (Long, in this case). We will use it to check if the
+ * RpcMetricsWrapper class properly sets values in the actual metrics record.
+ */
+class DummyMetricsRecord implements MetricsRecord {
+ private HashMap<String, Long> metricMap = new HashMap<String, Long>();
+
+ /** Do not need to implement these functions */
+ public String getRecordName() { return ""; }
+ public void setTag(String key, String value) { }
+ public void setTag(String tagName, int tagValue) {}
+ public void setTag(String tagName, long tagValue) {}
+ public void setTag(String tagName, short tagValue) {}
+ public void setTag(String tagName, byte tagValue) {}
+ public void removeTag(String tagName) {}
+ public void setMetric(String metricName, int metricValue) {}
+ public void setMetric(String metricName, short metricValue) {}
+ public void setMetric(String metricName, byte metricValue) {}
+ public void setMetric(String metricName, float metricValue) {}
+ public void incrMetric(String metricName, long metricValue) {}
+ public void incrMetric(String metricName, short metricValue) {}
+ public void incrMetric(String metricName, byte metricValue) {}
+ public void incrMetric(String metricName, float metricValue) {}
+ public void update() {}
+ public void remove() {}
+
+ /**
+ * Get the metric value
+ * @param metricName
+ * @return
+ */
+ public Long getMetric(String metricName) {
+ return metricMap.get(metricName);
+ }
+
+ /**
+ * Set the metric value
+ * @param metricName
+ * @param metricValue
+ */
+ public void setMetric(String metricName, long metricValue) {
+ metricMap.put(metricName, Long.valueOf(metricValue));
+ }
+
+ /**
+ * Increment the metric
+ * @param metricName
+ * @param metricValue
+ */
+ public void incrMetric(String metricName, int metricValue) {
+ Long value = getMetric(metricName);
+ if (value == null) {
+ setMetric(metricName, (long)metricValue);
+ } else {
+ setMetric(metricName, value.longValue() + (long)metricValue);
+ }
+ }
+}
+
+public class TestRpcMetricWrapper extends TestCase {
+ private MetricsRegistry registry = new MetricsRegistry();
+
+ /**
+ * Test if the RpcMetricWrapper pushes the metric name properly to the
+ * metric record object.
+ */
+ public void testRpcMetricPush() {
+ String metricName = "foo";
+ DummyMetricsRecord dummyMetricsRecord = new DummyMetricsRecord();
+ RpcMetricWrapper rpcMetricWrapper =
+ new RpcMetricWrapper(metricName, registry);
+
+ for (int i = 1; i <= 100; i++) {
+ rpcMetricWrapper.inc(i);
+ }
+ rpcMetricWrapper.pushMetric(dummyMetricsRecord);
+
+ assertEquals(50, dummyMetricsRecord.getMetric("foo_avg_time").longValue());
+ assertEquals(1, dummyMetricsRecord.getMetric("foo_min").longValue());
+ assertEquals(100, dummyMetricsRecord.getMetric("foo_max").longValue());
+ assertEquals(100, dummyMetricsRecord.getMetric("foo_num_ops").longValue());
+ assertEquals(95, dummyMetricsRecord.getMetric("foo_p95").longValue());
+ }
+}