You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2016/02/24 23:47:21 UTC
[1/3] hbase git commit: HBASE-12133 Add FastLongHistogram for metric
computation (Yi Deng)
Repository: hbase
Updated Branches:
refs/heads/branch-1 bcbe174a2 -> 3352173ec
HBASE-12133 Add FastLongHistogram for metric computation (Yi Deng)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b4f747f5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b4f747f5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b4f747f5
Branch: refs/heads/branch-1
Commit: b4f747f52e24afcf539a313a3bd2243ccf89b6b1
Parents: bcbe174
Author: stack <st...@apache.org>
Authored: Thu Oct 2 10:38:56 2014 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Wed Feb 24 14:35:22 2016 -0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/util/AtomicUtils.java | 63 +++++
.../hadoop/hbase/util/FastLongHistogram.java | 233 +++++++++++++++++++
.../hbase/util/TestFastLongHistogram.java | 99 ++++++++
3 files changed, 395 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4f747f5/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java
new file mode 100644
index 0000000..35391ee
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AtomicUtils.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * Utilities related to atomic operations.
+ */
+@InterfaceAudience.Private
+public class AtomicUtils {
+ /**
+ * Updates a AtomicLong which is supposed to maintain the minimum values. This method is not
+ * synchronized but is thread-safe.
+ */
+ public static void updateMin(AtomicLong min, long value) {
+ while (true) {
+ long cur = min.get();
+ if (value >= cur) {
+ break;
+ }
+
+ if (min.compareAndSet(cur, value)) {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Updates a AtomicLong which is supposed to maintain the maximum values. This method is not
+ * synchronized but is thread-safe.
+ */
+ public static void updateMax(AtomicLong max, long value) {
+ while (true) {
+ long cur = max.get();
+ if (value <= cur) {
+ break;
+ }
+
+ if (max.compareAndSet(cur, value)) {
+ break;
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4f747f5/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java
new file mode 100644
index 0000000..623cbdb
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicLongArray;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * FastLongHistogram is a thread-safe class that estimate distribution of data and computes the
+ * quantiles.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class FastLongHistogram {
+ /**
+ * Bins is a class containing a list of buckets(or bins) for estimation histogram of some data.
+ */
+ private static class Bins {
+ private final AtomicLongArray counts;
+ // inclusive
+ private final long binsMin;
+ // exclusive
+ private final long binsMax;
+ private final long bins10XMax;
+ private final AtomicLong min = new AtomicLong(Long.MAX_VALUE);
+ private final AtomicLong max = new AtomicLong(0L);
+ // set to true when any of data has been inserted to the Bins. It is set after the counts are
+ // updated.
+ private final AtomicBoolean hasData = new AtomicBoolean(false);
+
+ /**
+ * The constructor for creating a Bins without any prior data.
+ */
+ public Bins() {
+ this.counts = new AtomicLongArray(4);
+ this.binsMin = 0L;
+ this.binsMax = Long.MAX_VALUE;
+ this.bins10XMax = Long.MAX_VALUE;
+ }
+
+ /**
+ * The constructor for creating a Bins with last Bins.
+ * @param last the last Bins instance.
+ * @param quantiles the quantiles for creating the bins of the histogram.
+ */
+ public Bins(Bins last, int numOfBins, double minQ, double maxQ) {
+ long[] values = last.getQuantiles(new double[] { minQ, maxQ });
+ long wd = values[1] - values[0] + 1;
+ // expand minQ and maxQ in two ends back assuming uniform distribution
+ this.binsMin = Math.max(0L, (long) (values[0] - wd * minQ));
+ long binsMax = (long) (values[1] + wd * (1 - maxQ)) + 1;
+ // make sure each of bins is at least of width 1
+ this.binsMax = Math.max(binsMax, this.binsMin + numOfBins);
+ this.bins10XMax = Math.max((long) (values[1] + (binsMax - 1) * 9), this.binsMax + 1);
+
+ this.counts = new AtomicLongArray(numOfBins + 3);
+ }
+
+ /**
+ * Adds a value to the histogram.
+ */
+ public void add(long value, long count) {
+ AtomicUtils.updateMin(min, value);
+ AtomicUtils.updateMax(max, value);
+
+ if (value < this.binsMin) {
+ this.counts.addAndGet(0, count);
+ } else if (value > this.bins10XMax) {
+ this.counts.addAndGet(this.counts.length() - 1, count);
+ } else if (value >= this.binsMax) {
+ this.counts.addAndGet(this.counts.length() - 2, count);
+ } else {
+ // compute the position
+ int pos =
+ 1 + (int) ((value - this.binsMin) * (this.counts.length() - 3) / (this.binsMax - this.binsMin));
+ this.counts.addAndGet(pos, count);
+ }
+
+ // hasData needs to be updated as last
+ this.hasData.set(true);
+ }
+
+ /**
+ * Computes the quantiles give the ratios.
+ * @param smooth set to true to have a prior on the distribution. Used for recreating the bins.
+ */
+ public long[] getQuantiles(double[] quantiles) {
+ if (!this.hasData.get()) {
+ // No data yet.
+ return new long[quantiles.length];
+ }
+
+ // Make a snapshot of lowerCounter, higherCounter and bins.counts to counts.
+ // This is not synchronized, but since the counter are accumulating, the result is a good
+ // estimation of a snapshot.
+ long[] counts = new long[this.counts.length()];
+ long total = 0L;
+ for (int i = 0; i < this.counts.length(); i++) {
+ counts[i] = this.counts.get(i);
+ total += counts[i];
+ }
+
+ int rIndex = 0;
+ double qCount = total * quantiles[0];
+ long cum = 0L;
+
+ long[] res = new long[quantiles.length];
+ countsLoop: for (int i = 0; i < counts.length; i++) {
+ // mn and mx define a value range
+ long mn, mx;
+ if (i == 0) {
+ mn = this.min.get();
+ mx = this.binsMin;
+ } else if (i == counts.length - 1) {
+ mn = this.bins10XMax;
+ mx = this.max.get();
+ } else if (i == counts.length - 2) {
+ mn = this.binsMax;
+ mx = this.bins10XMax;
+ } else {
+ mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length() - 3);
+ mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length() - 3);
+ }
+
+ if (mx < this.min.get()) {
+ continue;
+ }
+ if (mn > this.max.get()) {
+ break;
+ }
+ mn = Math.max(mn, this.min.get());
+ mx = Math.min(mx, this.max.get());
+
+ // lastCum/cum are the corresponding counts to mn/mx
+ double lastCum = cum;
+ cum += counts[i];
+
+ // fill the results for qCount is within current range.
+ while (qCount <= cum) {
+ if (cum == lastCum) {
+ res[rIndex] = mn;
+ } else {
+ res[rIndex] = (long) ((qCount - lastCum) * (mx - mn) / (cum - lastCum) + mn);
+ }
+
+ // move to next quantile
+ rIndex++;
+ if (rIndex >= quantiles.length) {
+ break countsLoop;
+ }
+ qCount = total * quantiles[rIndex];
+ }
+ }
+ // In case quantiles contains values >= 100%
+ for (; rIndex < quantiles.length; rIndex++) {
+ res[rIndex] = this.max.get();
+ }
+
+ return res;
+ }
+ }
+
+ // The bins counting values. It is replaced with a new one in calling of reset().
+ private volatile Bins bins = new Bins();
+ // The quantiles for creating a Bins with last Bins.
+ private final int numOfBins;
+
+ /**
+ * Constructor.
+ * @param numOfBins the number of bins for the histogram. A larger value results in more precise
+ * results but with lower efficiency, and vice versus.
+ */
+ public FastLongHistogram(int numOfBins) {
+ this.numOfBins = numOfBins;
+ }
+
+ /**
+ * Constructor setting the bins assuming a uniform distribution within a range.
+ * @param numOfBins the number of bins for the histogram. A larger value results in more precise
+ * results but with lower efficiency, and vice versus.
+ * @param min lower bound of the region, inclusive.
+ * @param max higher bound of the region, inclusive.
+ */
+ public FastLongHistogram(int numOfBins, long min, long max) {
+ this(numOfBins);
+ Bins bins = new Bins();
+ bins.add(min, 1);
+ bins.add(max, 1);
+ this.bins = new Bins(bins, numOfBins, 0.01, 0.99);
+ }
+
+ /**
+ * Adds a value to the histogram.
+ */
+ public void add(long value, long count) {
+ this.bins.add(value, count);
+ }
+
+ /**
+ * Computes the quantiles give the ratios.
+ */
+ public long[] getQuantiles(double[] quantiles) {
+ return this.bins.getQuantiles(quantiles);
+ }
+
+ /**
+ * Resets the histogram for new counting.
+ */
+ public void reset() {
+ if (this.bins.hasData.get()) {
+ this.bins = new Bins(this.bins, numOfBins, 0.01, 0.99);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4f747f5/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java
new file mode 100644
index 0000000..e0437b5
--- /dev/null
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.Arrays;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Testcases for FastLongHistogram.
+ */
+@Category({ SmallTests.class })
+public class TestFastLongHistogram {
+
+ private static void doTestUniform(FastLongHistogram hist) {
+ long[] VALUES = { 0, 10, 20, 30, 40, 50 };
+ double[] qs = new double[VALUES.length];
+ for (int i = 0; i < qs.length; i++) {
+ qs[i] = (double) VALUES[i] / VALUES[VALUES.length - 1];
+ }
+
+ for (int i = 0; i < 10; i++) {
+ for (long v : VALUES) {
+ hist.add(v, 1);
+ }
+ long[] vals = hist.getQuantiles(qs);
+ System.out.println(Arrays.toString(vals));
+ for (int j = 0; j < qs.length; j++) {
+ Assert.assertTrue(j + "-th element org: " + VALUES[j] + ", act: " + vals[j],
+ Math.abs(vals[j] - VALUES[j]) <= 10);
+ }
+ hist.reset();
+ }
+ }
+
+ @Test
+ public void testUniform() {
+ FastLongHistogram hist = new FastLongHistogram(100, 0, 50);
+ doTestUniform(hist);
+ }
+
+ @Test
+ public void testAdaptionOfChange() {
+ // assumes the uniform distribution
+ FastLongHistogram hist = new FastLongHistogram(100, 0, 100);
+
+ Random rand = new Random();
+
+ for (int n = 0; n < 10; n++) {
+ for (int i = 0; i < 900; i++) {
+ hist.add(rand.nextInt(100), 1);
+ }
+
+ // add 10% outliers, this breaks the assumption, hope bin10xMax works
+ for (int i = 0; i < 100; i++) {
+ hist.add(1000 + rand.nextInt(100), 1);
+ }
+
+ long[] vals = hist.getQuantiles(new double[] { 0.25, 0.75, 0.95 });
+ System.out.println(Arrays.toString(vals));
+ if (n == 0) {
+ Assert.assertTrue("Out of possible value", vals[0] >= 0 && vals[0] <= 50);
+ Assert.assertTrue("Out of possible value", vals[1] >= 50 && vals[1] <= 100);
+ Assert.assertTrue("Out of possible value", vals[2] >= 900 && vals[2] <= 1100);
+ }
+
+ hist.reset();
+ }
+ }
+
+ @Test
+ public void testSameValues() {
+ FastLongHistogram hist = new FastLongHistogram(100);
+
+ hist.add(50, 100);
+
+ hist.reset();
+ doTestUniform(hist);
+ }
+}
[2/3] hbase git commit: HBASE-15222 Use less contended classes for
metrics
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java
index 32d4fae..aaf4359 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.metrics2.lib;
-import java.util.concurrent.atomic.AtomicLongArray;
-
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.metrics2.MetricsInfo;
@@ -28,31 +26,30 @@ import org.apache.hadoop.metrics2.MetricsInfo;
*/
@InterfaceAudience.Private
public class MutableTimeHistogram extends MutableRangeHistogram {
- private final String rangeType = "TimeRangeCount";
- private final long[] ranges =
+ private final static String RANGE_TYPE = "TimeRangeCount";
+ private final static long[] RANGES =
{ 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 60000, 120000, 300000, 600000 };
- private final AtomicLongArray rangeVals = new AtomicLongArray(ranges.length+1);
public MutableTimeHistogram(MetricsInfo info) {
this(info.name(), info.description());
}
public MutableTimeHistogram(String name, String description) {
- super(name, description);
+ this(name, description, RANGES[RANGES.length - 2]);
+ }
+
+ public MutableTimeHistogram(String name, String description, long expectedMax) {
+ super(name, description, expectedMax);
}
@Override
public String getRangeType() {
- return rangeType;
+ return RANGE_TYPE;
}
@Override
- public long[] getRange() {
- return ranges;
+ public long[] getRanges() {
+ return RANGES;
}
- @Override
- public AtomicLongArray getRangeVals() {
- return rangeVals;
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java
index 7381fb9..2e374f7 100644
--- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java
+++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.metrics;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.testclassification.MetricsTests;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -72,9 +73,9 @@ public class TestBaseSourceImpl {
@Test
public void testIncCounters() throws Exception {
bmsi.incCounters("testinccounter", 100);
- assertEquals(100, ((MutableCounterLong) bmsi.metricsRegistry.get("testinccounter")).value());
+ assertEquals(100, ((MutableFastCounter) bmsi.metricsRegistry.get("testinccounter")).value());
bmsi.incCounters("testinccounter", 100);
- assertEquals(200, ((MutableCounterLong) bmsi.metricsRegistry.get("testinccounter")).value());
+ assertEquals(200, ((MutableFastCounter) bmsi.metricsRegistry.get("testinccounter")).value());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
index 6986f12..3dcd5e2 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
@@ -178,7 +178,6 @@ org.apache.hadoop.util.StringUtils;
AgeSnapshot ageAtEvictionSnapshot = bc.getStats().getAgeAtEvictionSnapshot();
// Only show if non-zero mean and stddev as is the case in combinedblockcache
double mean = ageAtEvictionSnapshot.getMean();
- double stddev = ageAtEvictionSnapshot.getStdDev();
</%java>
<tr>
<td>Evicted</td>
@@ -197,13 +196,6 @@ org.apache.hadoop.util.StringUtils;
<td>Mean age of Blocks at eviction time (seconds)</td>
</tr>
</%if>
-<%if stddev > 0 %>
- <tr>
- <td>StdDev</td>
- <td><% String.format("%,d", (long)(ageAtEvictionSnapshot.getStdDev()/1000000)) %></td>
- <td>Standard Deviation for age of Blocks at eviction time</td>
- </tr>
-</%if>
</%def>
<%def hits_tmpl>
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon
index 523d1b9..fa55f6a 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon
@@ -37,7 +37,6 @@ org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator;
org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.Bucket;
org.apache.hadoop.util.StringUtils;
-com.yammer.metrics.stats.Snapshot;
</%import>
<%java>
BlockCache bc = cacheConfig == null ? null : cacheConfig.getBlockCache();
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
index ed0926f..e04756d 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
@@ -34,7 +34,6 @@ org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
org.apache.hadoop.hbase.util.DirectMemoryUtils;
org.apache.hadoop.util.StringUtils;
-com.yammer.metrics.stats.Snapshot;
java.lang.management.ManagementFactory;
</%import>
<div class="tabbable">
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java
index 24a4e32..4c1ad23 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java
@@ -17,58 +17,54 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import org.apache.hadoop.hbase.util.FastLongHistogram;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import com.yammer.metrics.core.Histogram;
-import com.yammer.metrics.stats.Snapshot;
-
/**
* Snapshot of block cache age in cache.
* This object is preferred because we can control how it is serialized out when JSON'ing.
*/
@JsonIgnoreProperties({"ageHistogram", "snapshot"})
public class AgeSnapshot {
- private final Histogram ageHistogram;
- private final Snapshot snapshot;
- AgeSnapshot(final Histogram ageHistogram) {
+ private final FastLongHistogram ageHistogram;
+ private final long[] quantiles;
+
+ AgeSnapshot(final FastLongHistogram ageHistogram) {
this.ageHistogram = ageHistogram;
- this.snapshot = ageHistogram.getSnapshot();
+ this.quantiles = ageHistogram.getQuantiles(new double[]{0.75, 0.95, 0.98, 0.99, 0.999});
}
public double get75thPercentile() {
- return snapshot.get75thPercentile();
+ return quantiles[0];
}
public double get95thPercentile() {
- return snapshot.get95thPercentile();
+ return quantiles[1];
}
public double get98thPercentile() {
- return snapshot.get98thPercentile();
+ return quantiles[2];
}
- public double get999thPercentile() {
- return snapshot.get999thPercentile();
+ public double get99thPercentile() {
+ return quantiles[3];
}
- public double get99thPercentile() {
- return snapshot.get99thPercentile();
+ public double get999thPercentile() {
+ return quantiles[4];
}
+
public double getMean() {
- return this.ageHistogram.mean();
+ return this.ageHistogram.getMean();
}
public double getMax() {
- return ageHistogram.max();
+ return this.ageHistogram.getMax();
}
public double getMin() {
- return ageHistogram.min();
- }
-
- public double getStdDev() {
- return ageHistogram.stdDev();
+ return this.ageHistogram.getMin();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
index 94638da..ff67337 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java
@@ -25,16 +25,13 @@ import java.util.concurrent.ConcurrentSkipListSet;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.FastLongHistogram;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.SerializationConfig;
-import com.yammer.metrics.core.Histogram;
-import com.yammer.metrics.core.MetricsRegistry;
-import com.yammer.metrics.stats.Snapshot;
-
/**
* Utilty for aggregating counts in CachedBlocks and toString/toJSON CachedBlocks and BlockCaches.
* No attempt has been made at making this thread safe.
@@ -42,11 +39,6 @@ import com.yammer.metrics.stats.Snapshot;
@InterfaceAudience.Private
public class BlockCacheUtil {
/**
- * Needed making histograms.
- */
- private static final MetricsRegistry METRICS = new MetricsRegistry();
-
- /**
* Needed generating JSON.
*/
private static final ObjectMapper MAPPER = new ObjectMapper();
@@ -204,7 +196,7 @@ public class BlockCacheUtil {
*/
private NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile =
new ConcurrentSkipListMap<String, NavigableSet<CachedBlock>>();
- Histogram age = METRICS.newHistogram(CachedBlocksByFile.class, "age");
+ FastLongHistogram hist = new FastLongHistogram();
/**
* @param cb
@@ -226,7 +218,7 @@ public class BlockCacheUtil {
this.dataSize += cb.getSize();
}
long age = this.now - cb.getCachedTime();
- this.age.update(age);
+ this.hist.add(age, 1);
return false;
}
@@ -269,18 +261,22 @@ public class BlockCacheUtil {
}
public AgeSnapshot getAgeInCacheSnapshot() {
- return new AgeSnapshot(this.age);
+ return new AgeSnapshot(this.hist);
}
@Override
public String toString() {
- Snapshot snapshot = this.age.getSnapshot();
- return "count=" + count + ", dataBlockCount=" + this.dataBlockCount + ", size=" + size +
+ AgeSnapshot snapshot = getAgeInCacheSnapshot();
+ return "count=" + count + ", dataBlockCount=" + dataBlockCount + ", size=" + size +
", dataSize=" + getDataSize() +
- ", mean age=" + this.age.mean() + ", stddev age=" + this.age.stdDev() +
- ", min age=" + this.age.min() + ", max age=" + this.age.max() +
- ", 95th percentile age=" + snapshot.get95thPercentile() +
- ", 99th percentile age=" + snapshot.get99thPercentile();
+ ", mean age=" + snapshot.getMean() +
+ ", min age=" + snapshot.getMin() +
+ ", max age=" + snapshot.getMax() +
+ ", 75th percentile age=" + snapshot.get75thPercentile() +
+ ", 95th percentile age=" + snapshot.get95thPercentile() +
+ ", 98th percentile age=" + snapshot.get98thPercentile() +
+ ", 99th percentile age=" + snapshot.get99thPercentile() +
+ ", 99.9th percentile age=" + snapshot.get99thPercentile();
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
index fff6585..b2a0d09 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java
@@ -22,18 +22,14 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import com.yammer.metrics.core.Histogram;
-import com.yammer.metrics.core.MetricsRegistry;
+import org.apache.hadoop.hbase.util.Counter;
+import org.apache.hadoop.hbase.util.FastLongHistogram;
/**
* Class that implements cache metrics.
*/
@InterfaceAudience.Private
public class CacheStats {
- /**
- * Needed making histograms.
- */
- private static final MetricsRegistry METRICS = new MetricsRegistry();
/** Sliding window statistics. The number of metric periods to include in
* sliding window hit ratio calculations.
@@ -41,10 +37,10 @@ public class CacheStats {
static final int DEFAULT_WINDOW_PERIODS = 5;
/** The number of getBlock requests that were cache hits */
- private final AtomicLong hitCount = new AtomicLong(0);
+ private final Counter hitCount = new Counter();
/** The number of getBlock requests that were cache hits from primary replica */
- private final AtomicLong primaryHitCount = new AtomicLong(0);
+ private final Counter primaryHitCount = new Counter();
/**
* The number of getBlock requests that were cache hits, but only from
@@ -52,27 +48,27 @@ public class CacheStats {
* attempt to read from the block cache even if they will not put new blocks
* into the block cache. See HBASE-2253 for more information.
*/
- private final AtomicLong hitCachingCount = new AtomicLong(0);
+ private final Counter hitCachingCount = new Counter();
/** The number of getBlock requests that were cache misses */
- private final AtomicLong missCount = new AtomicLong(0);
+ private final Counter missCount = new Counter();
/** The number of getBlock requests for primary replica that were cache misses */
- private final AtomicLong primaryMissCount = new AtomicLong(0);
+ private final Counter primaryMissCount = new Counter();
/**
* The number of getBlock requests that were cache misses, but only from
* requests that were set to use the block cache.
*/
- private final AtomicLong missCachingCount = new AtomicLong(0);
+ private final Counter missCachingCount = new Counter();
/** The number of times an eviction has occurred */
- private final AtomicLong evictionCount = new AtomicLong(0);
+ private final Counter evictionCount = new Counter();
/** The total number of blocks that have been evicted */
- private final AtomicLong evictedBlockCount = new AtomicLong(0);
+ private final Counter evictedBlockCount = new Counter();
/** The total number of blocks for primary replica that have been evicted */
- private final AtomicLong primaryEvictedBlockCount = new AtomicLong(0);
+ private final Counter primaryEvictedBlockCount = new Counter();
/** The total number of blocks that were not inserted. */
private final AtomicLong failedInserts = new AtomicLong(0);
@@ -100,7 +96,7 @@ public class CacheStats {
/**
* Keep running age at eviction time
*/
- private Histogram ageAtEviction;
+ private FastLongHistogram ageAtEviction;
private long startTime = System.nanoTime();
public CacheStats(final String name) {
@@ -113,7 +109,7 @@ public class CacheStats {
this.hitCachingCounts = initializeZeros(numPeriodsInWindow);
this.requestCounts = initializeZeros(numPeriodsInWindow);
this.requestCachingCounts = initializeZeros(numPeriodsInWindow);
- this.ageAtEviction = METRICS.newHistogram(CacheStats.class, name + ".ageAtEviction");
+ this.ageAtEviction = new FastLongHistogram();
}
@Override
@@ -125,14 +121,13 @@ public class CacheStats {
", evictedBlockCount=" + getEvictedCount() +
", primaryMissCount=" + getPrimaryMissCount() +
", primaryHitCount=" + getPrimaryHitCount() +
- ", evictedAgeMean=" + snapshot.getMean() +
- ", evictedAgeStdDev=" + snapshot.getStdDev();
+ ", evictedAgeMean=" + snapshot.getMean();
}
public void miss(boolean caching, boolean primary) {
- missCount.incrementAndGet();
- if (primary) primaryMissCount.incrementAndGet();
- if (caching) missCachingCount.incrementAndGet();
+ missCount.increment();
+ if (primary) primaryMissCount.increment();
+ if (caching) missCachingCount.increment();
}
public void hit(boolean caching) {
@@ -140,20 +135,20 @@ public class CacheStats {
}
public void hit(boolean caching, boolean primary) {
- hitCount.incrementAndGet();
- if (primary) primaryHitCount.incrementAndGet();
- if (caching) hitCachingCount.incrementAndGet();
+ hitCount.increment();
+ if (primary) primaryHitCount.increment();
+ if (caching) hitCachingCount.increment();
}
public void evict() {
- evictionCount.incrementAndGet();
+ evictionCount.increment();
}
public void evicted(final long t, boolean primary) {
- if (t > this.startTime) this.ageAtEviction.update(t - this.startTime);
- this.evictedBlockCount.incrementAndGet();
+ if (t > this.startTime) this.ageAtEviction.add(t - this.startTime,1);
+ this.evictedBlockCount.increment();
if (primary) {
- primaryEvictedBlockCount.incrementAndGet();
+ primaryEvictedBlockCount.increment();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index a67bf8c..3fb9554 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -35,7 +35,6 @@ import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -62,6 +61,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
import org.apache.hadoop.hbase.util.BloomFilterWriter;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Counter;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.io.Writable;
@@ -179,17 +179,19 @@ public class HFile {
*/
public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
// For measuring number of checksum failures
- static final AtomicLong checksumFailures = new AtomicLong();
+ static final Counter checksumFailures = new Counter();
// for test purpose
- public static final AtomicLong dataBlockReadCnt = new AtomicLong(0);
+ public static final Counter dataBlockReadCnt = new Counter();
/**
* Number of checksum verification failures. It also
* clears the counter.
*/
public static final long getChecksumFailuresCount() {
- return checksumFailures.getAndSet(0);
+ long count = checksumFailures.get();
+ checksumFailures.set(0);
+ return count;
}
/** API required to write an {@link HFile} */
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index edccfb5..6cd7b20 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -1523,7 +1523,7 @@ public class HFileBlock implements Cacheable {
HFile.LOG.warn(msg);
throw new IOException(msg); // cannot happen case here
}
- HFile.checksumFailures.incrementAndGet(); // update metrics
+ HFile.checksumFailures.increment(); // update metrics
// If we have a checksum failure, we fall back into a mode where
// the next few reads use HDFS level checksums. We aim to make the
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
index a1b4c34..50200dc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
@@ -417,7 +417,7 @@ public class HFileReaderV2 extends AbstractHFileReader {
assert cachedBlock.isUnpacked() : "Packed block leak.";
if (cachedBlock.getBlockType().isData()) {
if (updateCacheMetrics) {
- HFile.dataBlockReadCnt.incrementAndGet();
+ HFile.dataBlockReadCnt.increment();
}
// Validate encoding type for data blocks. We include encoding
// type in the cache key, and we expect it to match on a cache hit.
@@ -456,7 +456,7 @@ public class HFileReaderV2 extends AbstractHFileReader {
}
if (updateCacheMetrics && hfileBlock.getBlockType().isData()) {
- HFile.dataBlockReadCnt.incrementAndGet();
+ HFile.dataBlockReadCnt.increment();
}
return unpacked;
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java
index 51e6268..ec1d4d9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java
@@ -18,10 +18,9 @@
*/
package org.apache.hadoop.hbase.io.hfile.bucket;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.io.hfile.CacheStats;
+import org.apache.hadoop.hbase.util.Counter;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
/**
@@ -29,8 +28,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
*/
@InterfaceAudience.Private
public class BucketCacheStats extends CacheStats {
- private final AtomicLong ioHitCount = new AtomicLong(0);
- private final AtomicLong ioHitTime = new AtomicLong(0);
+ private final Counter ioHitCount = new Counter(0);
+ private final Counter ioHitTime = new Counter(0);
private final static int nanoTime = 1000000;
private long lastLogTime = EnvironmentEdgeManager.currentTime();
@@ -45,8 +44,8 @@ public class BucketCacheStats extends CacheStats {
}
public void ioHit(long time) {
- ioHitCount.incrementAndGet();
- ioHitTime.addAndGet(time);
+ ioHitCount.increment();
+ ioHitTime.add(time);
}
public long getIOHitsPerSecond() {
[3/3] hbase git commit: HBASE-15222 Use less contended classes for
metrics
Posted by ec...@apache.org.
HBASE-15222 Use less contended classes for metrics
Summary:
Use less contended things for metrics.
For histogram which was the largest culprit we use FastLongHistogram
For atomic long where possible we now use counter.
Test Plan: unit tests
Reviewers:
Subscribers:
Differential Revision: https://reviews.facebook.net/D54381
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3352173e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3352173e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3352173e
Branch: refs/heads/branch-1
Commit: 3352173ec87debdfe115755768c3f9c0ca82bb0e
Parents: b4f747f
Author: Elliott Clark <ec...@apache.org>
Authored: Thu Feb 18 09:54:05 2016 -0800
Committer: Elliott Clark <ec...@apache.org>
Committed: Wed Feb 24 14:47:00 2016 -0800
----------------------------------------------------------------------
.../hadoop/hbase/util/FastLongHistogram.java | 162 ++++++++++++++-----
.../hbase/util/TestFastLongHistogram.java | 37 ++++-
.../apache/hadoop/hbase/metrics/BaseSource.java | 8 -
.../apache/hadoop/metrics2/MetricHistogram.java | 3 +
hbase-hadoop2-compat/pom.xml | 8 +-
.../hbase/ipc/MetricsHBaseServerSourceImpl.java | 50 +++---
.../MetricsAssignmentManagerSourceImpl.java | 10 +-
.../MetricsMasterFilesystemSourceImpl.java | 14 +-
.../hbase/master/MetricsMasterSourceImpl.java | 4 +-
.../hbase/master/MetricsSnapshotSourceImpl.java | 8 +-
.../balancer/MetricsBalancerSourceImpl.java | 8 +-
.../hadoop/hbase/metrics/BaseSourceImpl.java | 17 +-
.../MetricsRegionServerSourceImpl.java | 16 +-
.../regionserver/MetricsRegionSourceImpl.java | 24 +--
.../regionserver/wal/MetricsWALSourceImpl.java | 10 +-
.../MetricsReplicationGlobalSourceSource.java | 36 ++---
.../MetricsReplicationSinkSourceImpl.java | 16 +-
.../MetricsReplicationSourceSourceImpl.java | 36 ++---
.../hbase/rest/MetricsRESTSourceImpl.java | 38 ++---
.../thrift/MetricsThriftServerSourceImpl.java | 13 +-
.../metrics2/lib/DynamicMetricsRegistry.java | 103 ++----------
.../metrics2/lib/MetricMutableQuantiles.java | 154 ------------------
.../metrics2/lib/MetricsExecutorImpl.java | 2 +-
.../hadoop/metrics2/lib/MutableFastCounter.java | 60 +++++++
.../hadoop/metrics2/lib/MutableHistogram.java | 135 ++++++----------
.../metrics2/lib/MutableRangeHistogram.java | 75 ++++-----
.../metrics2/lib/MutableSizeHistogram.java | 25 ++-
.../metrics2/lib/MutableTimeHistogram.java | 23 ++-
.../hbase/metrics/TestBaseSourceImpl.java | 5 +-
.../tmpl/regionserver/BlockCacheTmpl.jamon | 8 -
.../tmpl/regionserver/BlockCacheViewTmpl.jamon | 1 -
.../tmpl/regionserver/ServerMetricsTmpl.jamon | 1 -
.../hadoop/hbase/io/hfile/AgeSnapshot.java | 38 ++---
.../hadoop/hbase/io/hfile/BlockCacheUtil.java | 32 ++--
.../hadoop/hbase/io/hfile/CacheStats.java | 53 +++---
.../org/apache/hadoop/hbase/io/hfile/HFile.java | 10 +-
.../hadoop/hbase/io/hfile/HFileBlock.java | 2 +-
.../hadoop/hbase/io/hfile/HFileReaderV2.java | 4 +-
.../hbase/io/hfile/bucket/BucketCacheStats.java | 11 +-
39 files changed, 563 insertions(+), 697 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java
index 623cbdb..78b2bf0 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.util;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicLongArray;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -31,11 +30,20 @@ import org.apache.hadoop.hbase.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class FastLongHistogram {
+
+ /**
+ * Default number of bins.
+ */
+ public static final int DEFAULT_NBINS = 255;
+
+ public static final double[] DEFAULT_QUANTILES =
+ new double[]{0.25, 0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 0.999};
+
/**
* Bins is a class containing a list of buckets(or bins) for estimation histogram of some data.
*/
private static class Bins {
- private final AtomicLongArray counts;
+ private final Counter[] counts;
// inclusive
private final long binsMin;
// exclusive
@@ -43,6 +51,10 @@ public class FastLongHistogram {
private final long bins10XMax;
private final AtomicLong min = new AtomicLong(Long.MAX_VALUE);
private final AtomicLong max = new AtomicLong(0L);
+
+ private final Counter count = new Counter(0);
+ private final Counter total = new Counter(0);
+
// set to true when any of data has been inserted to the Bins. It is set after the counts are
// updated.
private final AtomicBoolean hasData = new AtomicBoolean(false);
@@ -50,17 +62,18 @@ public class FastLongHistogram {
/**
* The constructor for creating a Bins without any prior data.
*/
- public Bins() {
- this.counts = new AtomicLongArray(4);
- this.binsMin = 0L;
- this.binsMax = Long.MAX_VALUE;
- this.bins10XMax = Long.MAX_VALUE;
+ public Bins(int numBins) {
+ counts = createCounters(numBins + 3);
+ this.binsMin = 1L;
+
+ // These two numbers are total guesses
+ // and should be treated as highly suspect.
+ this.binsMax = 1000;
+ this.bins10XMax = binsMax * 10;
}
-
+
/**
* The constructor for creating a Bins with last Bins.
- * @param last the last Bins instance.
- * @param quantiles the quantiles for creating the bins of the histogram.
*/
public Bins(Bins last, int numOfBins, double minQ, double maxQ) {
long[] values = last.getQuantiles(new double[] { minQ, maxQ });
@@ -72,28 +85,51 @@ public class FastLongHistogram {
this.binsMax = Math.max(binsMax, this.binsMin + numOfBins);
this.bins10XMax = Math.max((long) (values[1] + (binsMax - 1) * 9), this.binsMax + 1);
- this.counts = new AtomicLongArray(numOfBins + 3);
+ this.counts = createCounters(numOfBins + 3);
+ }
+
+ private Counter[] createCounters(int num) {
+ Counter[] counters = new Counter[num];
+ for (int i = 0; i < num; i++) {
+ counters[i] = new Counter();
+ }
+ return counters;
+ }
+
+ private int getIndex(long value) {
+ if (value < this.binsMin) {
+ return 0;
+ } else if (value > this.bins10XMax) {
+ return this.counts.length - 1;
+ } else if (value >= this.binsMax) {
+ return this.counts.length - 2;
+ }
+ // compute the position
+ return 1 + (int) ((value - this.binsMin) * (this.counts.length - 3) /
+ (this.binsMax - this.binsMin));
+
}
/**
* Adds a value to the histogram.
*/
public void add(long value, long count) {
+ if (value < 0) {
+ // The whole computation is completely thrown off if there are negative numbers
+ //
+ // Normally we would throw an IllegalArgumentException however this is the metrics
+ // system and it should be completely safe at all times.
+ // So silently throw it away.
+ return;
+ }
AtomicUtils.updateMin(min, value);
AtomicUtils.updateMax(max, value);
- if (value < this.binsMin) {
- this.counts.addAndGet(0, count);
- } else if (value > this.bins10XMax) {
- this.counts.addAndGet(this.counts.length() - 1, count);
- } else if (value >= this.binsMax) {
- this.counts.addAndGet(this.counts.length() - 2, count);
- } else {
- // compute the position
- int pos =
- 1 + (int) ((value - this.binsMin) * (this.counts.length() - 3) / (this.binsMax - this.binsMin));
- this.counts.addAndGet(pos, count);
- }
+ this.count.add(count);
+ this.total.add(value * count);
+
+ int pos = getIndex(value);
+ this.counts[pos].add(count);
// hasData needs to be updated as last
this.hasData.set(true);
@@ -101,7 +137,6 @@ public class FastLongHistogram {
/**
* Computes the quantiles give the ratios.
- * @param smooth set to true to have a prior on the distribution. Used for recreating the bins.
*/
public long[] getQuantiles(double[] quantiles) {
if (!this.hasData.get()) {
@@ -112,10 +147,10 @@ public class FastLongHistogram {
// Make a snapshot of lowerCounter, higherCounter and bins.counts to counts.
// This is not synchronized, but since the counter are accumulating, the result is a good
// estimation of a snapshot.
- long[] counts = new long[this.counts.length()];
+ long[] counts = new long[this.counts.length];
long total = 0L;
- for (int i = 0; i < this.counts.length(); i++) {
- counts[i] = this.counts.get(i);
+ for (int i = 0; i < this.counts.length; i++) {
+ counts[i] = this.counts[i].get();
total += counts[i];
}
@@ -137,8 +172,8 @@ public class FastLongHistogram {
mn = this.binsMax;
mx = this.bins10XMax;
} else {
- mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length() - 3);
- mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length() - 3);
+ mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length - 3);
+ mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length - 3);
}
if (mx < this.min.get()) {
@@ -177,12 +212,27 @@ public class FastLongHistogram {
return res;
}
+
+
+ long getNumAtOrBelow(long val) {
+ final int targetIndex = getIndex(val);
+ long totalToCurrentIndex = 0;
+ for (int i = 0; i <= targetIndex; i++) {
+ totalToCurrentIndex += this.counts[i].get();
+ }
+ return totalToCurrentIndex;
+ }
}
// The bins counting values. It is replaced with a new one in calling of reset().
- private volatile Bins bins = new Bins();
- // The quantiles for creating a Bins with last Bins.
- private final int numOfBins;
+ private volatile Bins bins;
+
+ /**
+ * Constructor.
+ */
+ public FastLongHistogram() {
+ this(DEFAULT_NBINS);
+ }
/**
* Constructor.
@@ -190,7 +240,7 @@ public class FastLongHistogram {
* results but with lower efficiency, and vice versus.
*/
public FastLongHistogram(int numOfBins) {
- this.numOfBins = numOfBins;
+ this.bins = new Bins(numOfBins);
}
/**
@@ -202,10 +252,14 @@ public class FastLongHistogram {
*/
public FastLongHistogram(int numOfBins, long min, long max) {
this(numOfBins);
- Bins bins = new Bins();
+ Bins bins = new Bins(numOfBins);
bins.add(min, 1);
bins.add(max, 1);
- this.bins = new Bins(bins, numOfBins, 0.01, 0.99);
+ this.bins = new Bins(bins, numOfBins, 0.01, 0.999);
+ }
+
+ private FastLongHistogram(Bins bins) {
+ this.bins = bins;
}
/**
@@ -222,12 +276,46 @@ public class FastLongHistogram {
return this.bins.getQuantiles(quantiles);
}
+ public long[] getQuantiles() {
+ return this.bins.getQuantiles(DEFAULT_QUANTILES);
+ }
+
+ public long getMin() {
+ return this.bins.min.get();
+ }
+
+ public long getMax() {
+ return this.bins.max.get();
+ }
+
+ public long getCount() {
+ return this.bins.count.get();
+ }
+
+ public long getMean() {
+ Bins bins = this.bins;
+ long count = bins.count.get();
+ long total = bins.total.get();
+ if (count == 0) {
+ return 0;
+ }
+ return total / count;
+ }
+
+ public long getNumAtOrBelow(long value) {
+ return this.bins.getNumAtOrBelow(value);
+ }
+
/**
* Resets the histogram for new counting.
*/
- public void reset() {
+ public FastLongHistogram reset() {
if (this.bins.hasData.get()) {
- this.bins = new Bins(this.bins, numOfBins, 0.01, 0.99);
+ Bins oldBins = this.bins;
+ this.bins = new Bins(this.bins, this.bins.counts.length - 3, 0.01, 0.99);
+ return new FastLongHistogram(oldBins);
}
+
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java
index e0437b5..7304e2d 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java
@@ -20,15 +20,18 @@ package org.apache.hadoop.hbase.util;
import java.util.Arrays;
import java.util.Random;
-import org.apache.hadoop.hbase.SmallTests;
+
+import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import static org.junit.Assert.assertEquals;
+
/**
* Testcases for FastLongHistogram.
*/
-@Category({ SmallTests.class })
+@Category(SmallTests.class)
public class TestFastLongHistogram {
private static void doTestUniform(FastLongHistogram hist) {
@@ -87,6 +90,36 @@ public class TestFastLongHistogram {
}
}
+
+ @Test
+ public void testGetNumAtOrBelow() {
+ long[] VALUES = { 1, 10, 20, 30, 40, 50 };
+
+ FastLongHistogram h = new FastLongHistogram();
+ for (long v : VALUES) {
+ for (int i = 0; i < 100; i++) {
+ h.add(v, 1);
+ }
+ }
+
+ h.add(Integer.MAX_VALUE, 1);
+
+ h.reset();
+
+ for (long v : VALUES) {
+ for (int i = 0; i < 100; i++) {
+ h.add(v, 1);
+ }
+ }
+ // Add something way out there to make sure it doesn't throw off the counts.
+ h.add(Integer.MAX_VALUE, 1);
+
+ assertEquals(100, h.getNumAtOrBelow(1));
+ assertEquals(200, h.getNumAtOrBelow(11));
+ assertEquals(601, h.getNumAtOrBelow(Long.MAX_VALUE));
+ }
+
+
@Test
public void testSameValues() {
FastLongHistogram hist = new FastLongHistogram(100);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java
index 3ab783a..f79aa9f 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java
@@ -80,14 +80,6 @@ public interface BaseSource {
/**
- * Add some value to a Quantile (An accurate histogram).
- *
- * @param name the name of the quantile
- * @param value the value to add to the quantile
- */
- void updateQuantile(String name, long value);
-
- /**
* Get the metrics context. For hadoop metrics2 system this is usually an all lowercased string.
* eg. regionserver, master, thriftserver
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java
index b759efb..9d24a23 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java
@@ -30,10 +30,13 @@ public interface MetricHistogram {
String MAX_METRIC_NAME = "_max";
String MEAN_METRIC_NAME = "_mean";
String MEDIAN_METRIC_NAME = "_median";
+ String TWENTY_FIFTH_PERCENTILE_METRIC_NAME = "_25th_percentile";
String SEVENTY_FIFTH_PERCENTILE_METRIC_NAME = "_75th_percentile";
String NINETIETH_PERCENTILE_METRIC_NAME = "_90th_percentile";
String NINETY_FIFTH_PERCENTILE_METRIC_NAME = "_95th_percentile";
+ String NINETY_EIGHTH_PERCENTILE_METRIC_NAME = "_98th_percentile";
String NINETY_NINETH_PERCENTILE_METRIC_NAME = "_99th_percentile";
+ String NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME = "_99.9th_percentile";
/**
* Add a single value to a histogram's stream of values.
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml
index df77668..386e570 100644
--- a/hbase-hadoop2-compat/pom.xml
+++ b/hbase-hadoop2-compat/pom.xml
@@ -162,6 +162,10 @@ limitations under the License.
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop-compat</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
@@ -182,10 +186,6 @@ limitations under the License.
<version>${hadoop-two.version}</version>
</dependency>
<dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
- <dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
index 487f9f5..48f57e9 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
@@ -21,11 +21,11 @@ package org.apache.hadoop.hbase.ipc;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
+import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.Interns;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableHistogram;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
@InterfaceAudience.Private
public class MetricsHBaseServerSourceImpl extends BaseSourceImpl
@@ -33,29 +33,29 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl
private final MetricsHBaseServerWrapper wrapper;
- private final MutableCounterLong authorizationSuccesses;
- private final MutableCounterLong authorizationFailures;
- private final MutableCounterLong authenticationSuccesses;
- private final MutableCounterLong authenticationFailures;
- private final MutableCounterLong authenticationFallbacks;
- private final MutableCounterLong sentBytes;
- private final MutableCounterLong receivedBytes;
-
- private final MutableCounterLong exceptions;
- private final MutableCounterLong exceptionsOOO;
- private final MutableCounterLong exceptionsBusy;
- private final MutableCounterLong exceptionsUnknown;
- private final MutableCounterLong exceptionsSanity;
- private final MutableCounterLong exceptionsNSRE;
- private final MutableCounterLong exceptionsMoved;
- private final MutableCounterLong exceptionsMultiTooLarge;
-
-
- private MutableHistogram queueCallTime;
- private MutableHistogram processCallTime;
- private MutableHistogram totalCallTime;
- private MutableHistogram requestSize;
- private MutableHistogram responseSize;
+ private final MutableFastCounter authorizationSuccesses;
+ private final MutableFastCounter authorizationFailures;
+ private final MutableFastCounter authenticationSuccesses;
+ private final MutableFastCounter authenticationFailures;
+ private final MutableFastCounter authenticationFallbacks;
+ private final MutableFastCounter sentBytes;
+ private final MutableFastCounter receivedBytes;
+
+ private final MutableFastCounter exceptions;
+ private final MutableFastCounter exceptionsOOO;
+ private final MutableFastCounter exceptionsBusy;
+ private final MutableFastCounter exceptionsUnknown;
+ private final MutableFastCounter exceptionsSanity;
+ private final MutableFastCounter exceptionsNSRE;
+ private final MutableFastCounter exceptionsMoved;
+ private final MutableFastCounter exceptionsMultiTooLarge;
+
+
+ private MetricHistogram queueCallTime;
+ private MetricHistogram processCallTime;
+ private MetricHistogram totalCallTime;
+ private MetricHistogram requestSize;
+ private MetricHistogram responseSize;
public MetricsHBaseServerSourceImpl(String metricsName,
String metricsDescription,
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java
index ccf1c1d..a2192d2 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java
@@ -20,17 +20,19 @@ package org.apache.hadoop.hbase.master;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
+import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
-import org.apache.hadoop.metrics2.lib.MutableHistogram;
@InterfaceAudience.Private
-public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implements MetricsAssignmentManagerSource {
+public class MetricsAssignmentManagerSourceImpl
+ extends BaseSourceImpl
+ implements MetricsAssignmentManagerSource {
private MutableGaugeLong ritGauge;
private MutableGaugeLong ritCountOverThresholdGauge;
private MutableGaugeLong ritOldestAgeGauge;
- private MutableHistogram assignTimeHisto;
- private MutableHistogram bulkAssignTimeHisto;
+ private MetricHistogram assignTimeHisto;
+ private MetricHistogram bulkAssignTimeHisto;
public MetricsAssignmentManagerSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java
index 28414ea..7e8542c 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java
@@ -20,15 +20,17 @@ package org.apache.hadoop.hbase.master;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
-import org.apache.hadoop.metrics2.lib.MutableHistogram;
+import org.apache.hadoop.metrics2.MetricHistogram;
@InterfaceAudience.Private
-public class MetricsMasterFilesystemSourceImpl extends BaseSourceImpl implements MetricsMasterFileSystemSource {
+public class MetricsMasterFilesystemSourceImpl
+ extends BaseSourceImpl
+ implements MetricsMasterFileSystemSource {
- private MutableHistogram splitSizeHisto;
- private MutableHistogram splitTimeHisto;
- private MutableHistogram metaSplitTimeHisto;
- private MutableHistogram metaSplitSizeHisto;
+ private MetricHistogram splitSizeHisto;
+ private MetricHistogram splitTimeHisto;
+ private MetricHistogram metaSplitTimeHisto;
+ private MetricHistogram metaSplitSizeHisto;
public MetricsMasterFilesystemSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
index b0ba66e..ea21967 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.Interns;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
/**
* Hadoop2 implementation of MetricsMasterSource.
@@ -35,7 +35,7 @@ public class MetricsMasterSourceImpl
extends BaseSourceImpl implements MetricsMasterSource {
private final MetricsMasterWrapper masterWrapper;
- private MutableCounterLong clusterRequestsCounter;
+ private MutableFastCounter clusterRequestsCounter;
public MetricsMasterSourceImpl(MetricsMasterWrapper masterWrapper) {
this(METRICS_NAME,
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java
index c2fc6b9..bcefda0 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java
@@ -20,14 +20,14 @@ package org.apache.hadoop.hbase.master;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
-import org.apache.hadoop.metrics2.lib.MutableHistogram;
+import org.apache.hadoop.metrics2.MetricHistogram;
@InterfaceAudience.Private
public class MetricsSnapshotSourceImpl extends BaseSourceImpl implements MetricsSnapshotSource {
- private MutableHistogram snapshotTimeHisto;
- private MutableHistogram snapshotCloneTimeHisto;
- private MutableHistogram snapshotRestoreTimeHisto;
+ private MetricHistogram snapshotTimeHisto;
+ private MetricHistogram snapshotCloneTimeHisto;
+ private MetricHistogram snapshotRestoreTimeHisto;
public MetricsSnapshotSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java
index da34df2..0a74630 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java
@@ -20,14 +20,14 @@ package org.apache.hadoop.hbase.master.balancer;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableHistogram;
+import org.apache.hadoop.metrics2.MetricHistogram;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
@InterfaceAudience.Private
public class MetricsBalancerSourceImpl extends BaseSourceImpl implements MetricsBalancerSource{
- private MutableHistogram blanceClusterHisto;
- private MutableCounterLong miscCount;
+ private MetricHistogram blanceClusterHisto;
+ private MutableFastCounter miscCount;
public MetricsBalancerSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java
index 6756a21..f843ec2 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java
@@ -24,8 +24,7 @@ import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.impl.JmxCacheBuster;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MetricMutableQuantiles;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.metrics2.lib.MutableHistogram;
import org.apache.hadoop.metrics2.source.JvmMetrics;
@@ -88,7 +87,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource {
* @param value the new value of the gauge.
*/
public void setGauge(String gaugeName, long value) {
- MutableGaugeLong gaugeInt = metricsRegistry.getLongGauge(gaugeName, value);
+ MutableGaugeLong gaugeInt = metricsRegistry.getGauge(gaugeName, value);
gaugeInt.set(value);
}
@@ -99,7 +98,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource {
* @param delta The amount to increment the gauge by.
*/
public void incGauge(String gaugeName, long delta) {
- MutableGaugeLong gaugeInt = metricsRegistry.getLongGauge(gaugeName, 0l);
+ MutableGaugeLong gaugeInt = metricsRegistry.getGauge(gaugeName, 0l);
gaugeInt.incr(delta);
}
@@ -110,7 +109,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource {
* @param delta the ammount to subtract from a gauge value.
*/
public void decGauge(String gaugeName, long delta) {
- MutableGaugeLong gaugeInt = metricsRegistry.getLongGauge(gaugeName, 0l);
+ MutableGaugeLong gaugeInt = metricsRegistry.getGauge(gaugeName, 0l);
gaugeInt.decr(delta);
}
@@ -121,7 +120,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource {
* @param delta the ammount to increment
*/
public void incCounters(String key, long delta) {
- MutableCounterLong counter = metricsRegistry.getLongCounter(key, 0l);
+ MutableFastCounter counter = metricsRegistry.getCounter(key, 0l);
counter.incr(delta);
}
@@ -132,12 +131,6 @@ public class BaseSourceImpl implements BaseSource, MetricsSource {
histo.add(value);
}
- @Override
- public void updateQuantile(String name, long value) {
- MetricMutableQuantiles histo = metricsRegistry.getQuantile(name);
- histo.add(value);
- }
-
/**
* Remove a named gauge.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 9ee2b29..4e66ac0 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.Interns;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
/**
* Hadoop2 implementation of MetricsRegionServerSource.
@@ -45,13 +45,13 @@ public class MetricsRegionServerSourceImpl
private final MetricHistogram replayHisto;
private final MetricHistogram scanNextHisto;
- private final MutableCounterLong slowPut;
- private final MutableCounterLong slowDelete;
- private final MutableCounterLong slowGet;
- private final MutableCounterLong slowIncrement;
- private final MutableCounterLong slowAppend;
- private final MutableCounterLong splitRequest;
- private final MutableCounterLong splitSuccess;
+ private final MutableFastCounter slowPut;
+ private final MutableFastCounter slowDelete;
+ private final MutableFastCounter slowGet;
+ private final MutableFastCounter slowIncrement;
+ private final MutableFastCounter slowAppend;
+ private final MutableFastCounter splitRequest;
+ private final MutableFastCounter splitSuccess;
private final MetricHistogram splitTimeHisto;
private final MetricHistogram flushTimeHisto;
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index eefab08..494d3de 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -23,11 +23,11 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
import org.apache.hadoop.metrics2.lib.Interns;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableHistogram;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
@InterfaceAudience.Private
public class MetricsRegionSourceImpl implements MetricsRegionSource {
@@ -53,12 +53,12 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
private final String regionAppendKey;
private final String regionScanNextKey;
- private final MutableCounterLong regionPut;
- private final MutableCounterLong regionDelete;
- private final MutableCounterLong regionIncrement;
- private final MutableCounterLong regionAppend;
- private final MutableHistogram regionGet;
- private final MutableHistogram regionScanNext;
+ private final MutableFastCounter regionPut;
+ private final MutableFastCounter regionDelete;
+ private final MutableFastCounter regionIncrement;
+ private final MutableFastCounter regionAppend;
+ private final MetricHistogram regionGet;
+ private final MetricHistogram regionScanNext;
private final int hashCode;
public MetricsRegionSourceImpl(MetricsRegionWrapper regionWrapper,
@@ -80,16 +80,16 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
String suffix = "Count";
regionPutKey = regionNamePrefix + MetricsRegionServerSource.MUTATE_KEY + suffix;
- regionPut = registry.getLongCounter(regionPutKey, 0L);
+ regionPut = registry.getCounter(regionPutKey, 0L);
regionDeleteKey = regionNamePrefix + MetricsRegionServerSource.DELETE_KEY + suffix;
- regionDelete = registry.getLongCounter(regionDeleteKey, 0L);
+ regionDelete = registry.getCounter(regionDeleteKey, 0L);
regionIncrementKey = regionNamePrefix + MetricsRegionServerSource.INCREMENT_KEY + suffix;
- regionIncrement = registry.getLongCounter(regionIncrementKey, 0L);
+ regionIncrement = registry.getCounter(regionIncrementKey, 0L);
regionAppendKey = regionNamePrefix + MetricsRegionServerSource.APPEND_KEY + suffix;
- regionAppend = registry.getLongCounter(regionAppendKey, 0L);
+ regionAppend = registry.getCounter(regionAppendKey, 0L);
regionGetKey = regionNamePrefix + MetricsRegionServerSource.GET_KEY;
regionGet = registry.newTimeHistogram(regionGetKey);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
index 90ac4c3..aec3278 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.regionserver.wal;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricHistogram;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
/**
@@ -36,10 +36,10 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
private final MetricHistogram appendSizeHisto;
private final MetricHistogram appendTimeHisto;
private final MetricHistogram syncTimeHisto;
- private final MutableCounterLong appendCount;
- private final MutableCounterLong slowAppendCount;
- private final MutableCounterLong logRollRequested;
- private final MutableCounterLong lowReplicationLogRollRequested;
+ private final MutableFastCounter appendCount;
+ private final MutableFastCounter slowAppendCount;
+ private final MutableFastCounter logRollRequested;
+ private final MutableFastCounter lowReplicationLogRollRequested;
public MetricsWALSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index 392cd39..93b10b6 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hbase.replication.regionserver;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
public class MetricsReplicationGlobalSourceSource implements MetricsReplicationSourceSource{
@@ -26,38 +26,38 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
private final MutableGaugeLong ageOfLastShippedOpGauge;
private final MutableGaugeLong sizeOfLogQueueGauge;
- private final MutableCounterLong logReadInEditsCounter;
- private final MutableCounterLong logEditsFilteredCounter;
- private final MutableCounterLong shippedBatchesCounter;
- private final MutableCounterLong shippedOpsCounter;
- private final MutableCounterLong shippedKBsCounter;
- private final MutableCounterLong logReadInBytesCounter;
- private final MutableCounterLong shippedHFilesCounter;
+ private final MutableFastCounter logReadInEditsCounter;
+ private final MutableFastCounter logEditsFilteredCounter;
+ private final MutableFastCounter shippedBatchesCounter;
+ private final MutableFastCounter shippedOpsCounter;
+ private final MutableFastCounter shippedKBsCounter;
+ private final MutableFastCounter logReadInBytesCounter;
+ private final MutableFastCounter shippedHFilesCounter;
private final MutableGaugeLong sizeOfHFileRefsQueueGauge;
public MetricsReplicationGlobalSourceSource(MetricsReplicationSourceImpl rms) {
this.rms = rms;
- ageOfLastShippedOpGauge = rms.getMetricsRegistry().getLongGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L);
+ ageOfLastShippedOpGauge = rms.getMetricsRegistry().getGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L);
- sizeOfLogQueueGauge = rms.getMetricsRegistry().getLongGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L);
+ sizeOfLogQueueGauge = rms.getMetricsRegistry().getGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L);
- shippedBatchesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_BATCHES, 0L);
+ shippedBatchesCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_BATCHES, 0L);
- shippedOpsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_OPS, 0L);
+ shippedOpsCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_OPS, 0L);
- shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_KBS, 0L);
+ shippedKBsCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_KBS, 0L);
- logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_BYTES, 0L);
+ logReadInBytesCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_READ_IN_BYTES, 0L);
- logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_EDITS, 0L);
+ logReadInEditsCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_READ_IN_EDITS, 0L);
- logEditsFilteredCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_EDITS_FILTERED, 0L);
+ logEditsFilteredCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_EDITS_FILTERED, 0L);
- shippedHFilesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_HFILES, 0L);
+ shippedHFilesCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_HFILES, 0L);
sizeOfHFileRefsQueueGauge =
- rms.getMetricsRegistry().getLongGauge(SOURCE_SIZE_OF_HFILE_REFS_QUEUE, 0L);
+ rms.getMetricsRegistry().getGauge(SOURCE_SIZE_OF_HFILE_REFS_QUEUE, 0L);
}
@Override public void setLastShippedAge(long age) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
index 8f4a337..540212a 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java
@@ -18,21 +18,21 @@
package org.apache.hadoop.hbase.replication.regionserver;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
public class MetricsReplicationSinkSourceImpl implements MetricsReplicationSinkSource {
private final MutableGaugeLong ageGauge;
- private final MutableCounterLong batchesCounter;
- private final MutableCounterLong opsCounter;
- private final MutableCounterLong hfilesCounter;
+ private final MutableFastCounter batchesCounter;
+ private final MutableFastCounter opsCounter;
+ private final MutableFastCounter hfilesCounter;
public MetricsReplicationSinkSourceImpl(MetricsReplicationSourceImpl rms) {
- ageGauge = rms.getMetricsRegistry().getLongGauge(SINK_AGE_OF_LAST_APPLIED_OP, 0L);
- batchesCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_BATCHES, 0L);
- opsCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_OPS, 0L);
- hfilesCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_HFILES, 0L);
+ ageGauge = rms.getMetricsRegistry().getGauge(SINK_AGE_OF_LAST_APPLIED_OP, 0L);
+ batchesCounter = rms.getMetricsRegistry().getCounter(SINK_APPLIED_BATCHES, 0L);
+ opsCounter = rms.getMetricsRegistry().getCounter(SINK_APPLIED_OPS, 0L);
+ hfilesCounter = rms.getMetricsRegistry().getCounter(SINK_APPLIED_HFILES, 0L);
}
@Override public void setLastAppliedOpAge(long age) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
index 217cc3e..9941712 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSourceSource {
@@ -37,13 +37,13 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
private final MutableGaugeLong ageOfLastShippedOpGauge;
private final MutableGaugeLong sizeOfLogQueueGauge;
- private final MutableCounterLong logReadInEditsCounter;
- private final MutableCounterLong logEditsFilteredCounter;
- private final MutableCounterLong shippedBatchesCounter;
- private final MutableCounterLong shippedOpsCounter;
- private final MutableCounterLong shippedKBsCounter;
- private final MutableCounterLong logReadInBytesCounter;
- private final MutableCounterLong shippedHFilesCounter;
+ private final MutableFastCounter logReadInEditsCounter;
+ private final MutableFastCounter logEditsFilteredCounter;
+ private final MutableFastCounter shippedBatchesCounter;
+ private final MutableFastCounter shippedOpsCounter;
+ private final MutableFastCounter shippedKBsCounter;
+ private final MutableFastCounter logReadInBytesCounter;
+ private final MutableFastCounter shippedHFilesCounter;
private final MutableGaugeLong sizeOfHFileRefsQueueGauge;
public MetricsReplicationSourceSourceImpl(MetricsReplicationSourceImpl rms, String id) {
@@ -51,34 +51,34 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
this.id = id;
ageOfLastShippedOpKey = "source." + id + ".ageOfLastShippedOp";
- ageOfLastShippedOpGauge = rms.getMetricsRegistry().getLongGauge(ageOfLastShippedOpKey, 0L);
+ ageOfLastShippedOpGauge = rms.getMetricsRegistry().getGauge(ageOfLastShippedOpKey, 0L);
sizeOfLogQueueKey = "source." + id + ".sizeOfLogQueue";
- sizeOfLogQueueGauge = rms.getMetricsRegistry().getLongGauge(sizeOfLogQueueKey, 0L);
+ sizeOfLogQueueGauge = rms.getMetricsRegistry().getGauge(sizeOfLogQueueKey, 0L);
shippedBatchesKey = "source." + this.id + ".shippedBatches";
- shippedBatchesCounter = rms.getMetricsRegistry().getLongCounter(shippedBatchesKey, 0L);
+ shippedBatchesCounter = rms.getMetricsRegistry().getCounter(shippedBatchesKey, 0L);
shippedOpsKey = "source." + this.id + ".shippedOps";
- shippedOpsCounter = rms.getMetricsRegistry().getLongCounter(shippedOpsKey, 0L);
+ shippedOpsCounter = rms.getMetricsRegistry().getCounter(shippedOpsKey, 0L);
shippedKBsKey = "source." + this.id + ".shippedKBs";
- shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(shippedKBsKey, 0L);
+ shippedKBsCounter = rms.getMetricsRegistry().getCounter(shippedKBsKey, 0L);
logReadInBytesKey = "source." + this.id + ".logReadInBytes";
- logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(logReadInBytesKey, 0L);
+ logReadInBytesCounter = rms.getMetricsRegistry().getCounter(logReadInBytesKey, 0L);
logReadInEditsKey = "source." + id + ".logEditsRead";
- logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(logReadInEditsKey, 0L);
+ logReadInEditsCounter = rms.getMetricsRegistry().getCounter(logReadInEditsKey, 0L);
logEditsFilteredKey = "source." + id + ".logEditsFiltered";
- logEditsFilteredCounter = rms.getMetricsRegistry().getLongCounter(logEditsFilteredKey, 0L);
+ logEditsFilteredCounter = rms.getMetricsRegistry().getCounter(logEditsFilteredKey, 0L);
shippedHFilesKey = "source." + this.id + ".shippedHFiles";
- shippedHFilesCounter = rms.getMetricsRegistry().getLongCounter(shippedHFilesKey, 0L);
+ shippedHFilesCounter = rms.getMetricsRegistry().getCounter(shippedHFilesKey, 0L);
sizeOfHFileRefsQueueKey = "source." + id + ".sizeOfHFileRefsQueue";
- sizeOfHFileRefsQueueGauge = rms.getMetricsRegistry().getLongGauge(sizeOfHFileRefsQueueKey, 0L);
+ sizeOfHFileRefsQueueGauge = rms.getMetricsRegistry().getGauge(sizeOfHFileRefsQueueKey, 0L);
}
@Override public void setLastShippedAge(long age) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java
index 9eae18b..71755b7 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.rest;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableFastCounter;
/**
* Hadoop Two implementation of a metrics2 source that will export metrics from the Rest server to
@@ -31,15 +31,15 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong;
@InterfaceAudience.Private
public class MetricsRESTSourceImpl extends BaseSourceImpl implements MetricsRESTSource {
- private MutableCounterLong request;
- private MutableCounterLong sucGet;
- private MutableCounterLong sucPut;
- private MutableCounterLong sucDel;
- private MutableCounterLong sucScan;
- private MutableCounterLong fGet;
- private MutableCounterLong fPut;
- private MutableCounterLong fDel;
- private MutableCounterLong fScan;
+ private MutableFastCounter request;
+ private MutableFastCounter sucGet;
+ private MutableFastCounter sucPut;
+ private MutableFastCounter sucDel;
+ private MutableFastCounter sucScan;
+ private MutableFastCounter fGet;
+ private MutableFastCounter fPut;
+ private MutableFastCounter fDel;
+ private MutableFastCounter fScan;
public MetricsRESTSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, CONTEXT, JMX_CONTEXT);
@@ -55,17 +55,17 @@ public class MetricsRESTSourceImpl extends BaseSourceImpl implements MetricsREST
@Override
public void init() {
super.init();
- request = getMetricsRegistry().getLongCounter(REQUEST_KEY, 0l);
+ request = getMetricsRegistry().getCounter(REQUEST_KEY, 0l);
- sucGet = getMetricsRegistry().getLongCounter(SUCCESSFUL_GET_KEY, 0l);
- sucPut = getMetricsRegistry().getLongCounter(SUCCESSFUL_PUT_KEY, 0l);
- sucDel = getMetricsRegistry().getLongCounter(SUCCESSFUL_DELETE_KEY, 0l);
- sucScan = getMetricsRegistry().getLongCounter(SUCCESSFUL_SCAN_KEY, 0L);
+ sucGet = getMetricsRegistry().getCounter(SUCCESSFUL_GET_KEY, 0l);
+ sucPut = getMetricsRegistry().getCounter(SUCCESSFUL_PUT_KEY, 0l);
+ sucDel = getMetricsRegistry().getCounter(SUCCESSFUL_DELETE_KEY, 0l);
+ sucScan = getMetricsRegistry().getCounter(SUCCESSFUL_SCAN_KEY, 0L);
- fGet = getMetricsRegistry().getLongCounter(FAILED_GET_KEY, 0l);
- fPut = getMetricsRegistry().getLongCounter(FAILED_PUT_KEY, 0l);
- fDel = getMetricsRegistry().getLongCounter(FAILED_DELETE_KEY, 0l);
- fScan = getMetricsRegistry().getLongCounter(FAILED_SCAN_KEY, 0l);
+ fGet = getMetricsRegistry().getCounter(FAILED_GET_KEY, 0l);
+ fPut = getMetricsRegistry().getCounter(FAILED_PUT_KEY, 0l);
+ fDel = getMetricsRegistry().getCounter(FAILED_DELETE_KEY, 0l);
+ fScan = getMetricsRegistry().getCounter(FAILED_SCAN_KEY, 0l);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java
index f9612e5..f5d83b1 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.thrift;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
+import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.metrics2.lib.MutableHistogram;
@@ -32,12 +33,12 @@ import org.apache.hadoop.metrics2.lib.MutableHistogram;
public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements
MetricsThriftServerSource {
- private MutableHistogram batchGetStat;
- private MutableHistogram batchMutateStat;
- private MutableHistogram queueTimeStat;
+ private MetricHistogram batchGetStat;
+ private MetricHistogram batchMutateStat;
+ private MetricHistogram queueTimeStat;
- private MutableHistogram thriftCallStat;
- private MutableHistogram thriftSlowCallStat;
+ private MetricHistogram thriftCallStat;
+ private MetricHistogram thriftSlowCallStat;
private MutableGaugeLong callQueueLenGauge;
@@ -56,7 +57,7 @@ public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements
queueTimeStat = getMetricsRegistry().newTimeHistogram(TIME_IN_QUEUE_KEY);
thriftCallStat = getMetricsRegistry().newTimeHistogram(THRIFT_CALL_KEY);
thriftSlowCallStat = getMetricsRegistry().newTimeHistogram(SLOW_THRIFT_CALL_KEY);
- callQueueLenGauge = getMetricsRegistry().getLongGauge(CALL_QUEUE_LEN_KEY, 0);
+ callQueueLenGauge = getMetricsRegistry().getGauge(CALL_QUEUE_LEN_KEY, 0);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java
index ee13c76..1f3da12 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java
@@ -106,35 +106,13 @@ public class DynamicMetricsRegistry {
}
/**
- * Create a mutable integer counter
- * @param name of the metric
- * @param desc metric description
- * @param iVal initial value
- * @return a new counter object
- */
- public MutableCounterInt newCounter(String name, String desc, int iVal) {
- return newCounter(new MetricsInfoImpl(name, desc), iVal);
- }
-
- /**
- * Create a mutable integer counter
- * @param info metadata of the metric
- * @param iVal initial value
- * @return a new counter object
- */
- public MutableCounterInt newCounter(MetricsInfo info, int iVal) {
- MutableCounterInt ret = new MutableCounterInt(info, iVal);
- return addNewMetricIfAbsent(info.name(), ret, MutableCounterInt.class);
- }
-
- /**
* Create a mutable long integer counter
* @param name of the metric
* @param desc metric description
* @param iVal initial value
* @return a new counter object
*/
- public MutableCounterLong newCounter(String name, String desc, long iVal) {
+ public MutableFastCounter newCounter(String name, String desc, long iVal) {
return newCounter(new MetricsInfoImpl(name, desc), iVal);
}
@@ -144,30 +122,9 @@ public class DynamicMetricsRegistry {
* @param iVal initial value
* @return a new counter object
*/
- public MutableCounterLong newCounter(MetricsInfo info, long iVal) {
- MutableCounterLong ret = new MutableCounterLong(info, iVal);
- return addNewMetricIfAbsent(info.name(), ret, MutableCounterLong.class);
- }
-
- /**
- * Create a mutable integer gauge
- * @param name of the metric
- * @param desc metric description
- * @param iVal initial value
- * @return a new gauge object
- */
- public MutableGaugeInt newGauge(String name, String desc, int iVal) {
- return newGauge(new MetricsInfoImpl(name, desc), iVal);
- }
- /**
- * Create a mutable integer gauge
- * @param info metadata of the metric
- * @param iVal initial value
- * @return a new gauge object
- */
- public MutableGaugeInt newGauge(MetricsInfo info, int iVal) {
- MutableGaugeInt ret = new MutableGaugeInt(info, iVal);
- return addNewMetricIfAbsent(info.name(), ret, MutableGaugeInt.class);
+ public MutableFastCounter newCounter(MetricsInfo info, long iVal) {
+ MutableFastCounter ret = new MutableFastCounter(info, iVal);
+ return addNewMetricIfAbsent(info.name(), ret, MutableFastCounter.class);
}
/**
@@ -326,19 +283,6 @@ public class DynamicMetricsRegistry {
return addNewMetricIfAbsent(name, histo, MutableSizeHistogram.class);
}
- /**
- * Create a new MutableQuantile(A more accurate histogram).
- * @param name The name of the histogram
- * @return a new MutableQuantile
- */
- public MetricMutableQuantiles newQuantile(String name) {
- return newQuantile(name, "");
- }
-
- public MetricMutableQuantiles newQuantile(String name, String desc) {
- MetricMutableQuantiles histo = new MetricMutableQuantiles(name, desc, "Ops", "", 60);
- return addNewMetricIfAbsent(name, histo, MetricMutableQuantiles.class);
- }
synchronized void add(String name, MutableMetric metric) {
addNewMetricIfAbsent(name, metric, MutableMetric.class);
@@ -475,7 +419,7 @@ public class DynamicMetricsRegistry {
* @param gaugeName name of the gauge to create or get.
* @param potentialStartingValue value of the new gauge if we have to create it.
*/
- public MutableGaugeLong getLongGauge(String gaugeName, long potentialStartingValue) {
+ public MutableGaugeLong getGauge(String gaugeName, long potentialStartingValue) {
//Try and get the guage.
MutableMetric metric = metricsMap.get(gaugeName);
@@ -510,12 +454,12 @@ public class DynamicMetricsRegistry {
* @param counterName Name of the counter to get
* @param potentialStartingValue starting value if we have to create a new counter
*/
- public MutableCounterLong getLongCounter(String counterName, long potentialStartingValue) {
- //See getLongGauge for description on how this works.
+ public MutableFastCounter getCounter(String counterName, long potentialStartingValue) {
+ //See getGauge for description on how this works.
MutableMetric counter = metricsMap.get(counterName);
if (counter == null) {
- MutableCounterLong newCounter =
- new MutableCounterLong(new MetricsInfoImpl(counterName, ""), potentialStartingValue);
+ MutableFastCounter newCounter =
+ new MutableFastCounter(new MetricsInfoImpl(counterName, ""), potentialStartingValue);
counter = metricsMap.putIfAbsent(counterName, newCounter);
if (counter == null) {
return newCounter;
@@ -523,16 +467,16 @@ public class DynamicMetricsRegistry {
}
- if (!(counter instanceof MutableCounterLong)) {
+ if (!(counter instanceof MutableCounter)) {
throw new MetricsException("Metric already exists in registry for metric name: " +
- counterName + " and not of type MetricMutableCounterLong");
+ counterName + " and not of type MutableCounter");
}
- return (MutableCounterLong) counter;
+ return (MutableFastCounter) counter;
}
public MutableHistogram getHistogram(String histoName) {
- //See getLongGauge for description on how this works.
+ //See getGauge for description on how this works.
MutableMetric histo = metricsMap.get(histoName);
if (histo == null) {
MutableHistogram newCounter =
@@ -552,27 +496,6 @@ public class DynamicMetricsRegistry {
return (MutableHistogram) histo;
}
- public MetricMutableQuantiles getQuantile(String histoName) {
- //See getLongGauge for description on how this works.
- MutableMetric histo = metricsMap.get(histoName);
- if (histo == null) {
- MetricMutableQuantiles newCounter =
- new MetricMutableQuantiles(histoName, "", "Ops", "", 60);
- histo = metricsMap.putIfAbsent(histoName, newCounter);
- if (histo == null) {
- return newCounter;
- }
- }
-
-
- if (!(histo instanceof MetricMutableQuantiles)) {
- throw new MetricsException("Metric already exists in registry for metric name: " +
- histoName + " and not of type MutableHistogram");
- }
-
- return (MetricMutableQuantiles) histo;
- }
-
private<T extends MutableMetric> T
addNewMetricIfAbsent(String name,
T ret,
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java
deleted file mode 100644
index c03654b..0000000
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics2.lib;
-
-import static org.apache.hadoop.metrics2.lib.Interns.info;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.metrics2.MetricHistogram;
-import org.apache.hadoop.metrics2.MetricsExecutor;
-import org.apache.hadoop.metrics2.MetricsInfo;
-import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.util.MetricQuantile;
-import org.apache.hadoop.metrics2.util.MetricSampleQuantiles;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Watches a stream of long values, maintaining online estimates of specific quantiles with provably
- * low error bounds. This is particularly useful for accurate high-percentile (e.g. 95th, 99th)
- * latency metrics.
- */
-@InterfaceAudience.Private
-public class MetricMutableQuantiles extends MutableMetric implements MetricHistogram {
-
- static final MetricQuantile[] quantiles = {new MetricQuantile(0.50, 0.050),
- new MetricQuantile(0.75, 0.025), new MetricQuantile(0.90, 0.010),
- new MetricQuantile(0.95, 0.005), new MetricQuantile(0.99, 0.001)};
-
- private final MetricsInfo numInfo;
- private final MetricsInfo[] quantileInfos;
- private final int interval;
-
- private MetricSampleQuantiles estimator;
- private long previousCount = 0;
- private MetricsExecutor executor;
-
-
- @VisibleForTesting
- protected Map<MetricQuantile, Long> previousSnapshot = null;
-
- /**
- * Instantiates a new {@link MetricMutableQuantiles} for a metric that rolls itself over on the
- * specified time interval.
- *
- * @param name of the metric
- * @param description long-form textual description of the metric
- * @param sampleName type of items in the stream (e.g., "Ops")
- * @param valueName type of the values
- * @param interval rollover interval (in seconds) of the estimator
- */
- public MetricMutableQuantiles(String name, String description, String sampleName,
- String valueName, int interval) {
- String ucName = StringUtils.capitalize(name);
- String usName = StringUtils.capitalize(sampleName);
- String uvName = StringUtils.capitalize(valueName);
- String desc = StringUtils.uncapitalize(description);
- String lsName = StringUtils.uncapitalize(sampleName);
- String lvName = StringUtils.uncapitalize(valueName);
-
- numInfo = info(ucName + "Num" + usName, String.format(
- "Number of %s for %s with %ds interval", lsName, desc, interval));
- // Construct the MetricsInfos for the quantiles, converting to percentiles
- quantileInfos = new MetricsInfo[quantiles.length];
- String nameTemplate = "%s%dthPercentile%dsInterval%s";
- String descTemplate = "%d percentile %s with %d second interval for %s";
- for (int i = 0; i < quantiles.length; i++) {
- int percentile = (int) (100 * quantiles[i].quantile);
- quantileInfos[i] = info(String.format(nameTemplate, ucName, percentile, interval, uvName),
- String.format(descTemplate, percentile, lvName, interval, desc));
- }
-
- estimator = new MetricSampleQuantiles(quantiles);
- executor = new MetricsExecutorImpl();
- this.interval = interval;
- executor.getExecutor().scheduleAtFixedRate(new RolloverSample(this),
- interval,
- interval,
- TimeUnit.SECONDS);
- }
-
- @Override
- public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) {
- if (all || changed()) {
- builder.addGauge(numInfo, previousCount);
- for (int i = 0; i < quantiles.length; i++) {
- long newValue = 0;
- // If snapshot is null, we failed to update since the window was empty
- if (previousSnapshot != null) {
- newValue = previousSnapshot.get(quantiles[i]);
- }
- builder.addGauge(quantileInfos[i], newValue);
- }
- if (changed()) {
- clearChanged();
- }
- }
- }
-
- public synchronized void add(long value) {
- estimator.insert(value);
- }
-
- public int getInterval() {
- return interval;
- }
-
- /** Runnable used to periodically roll over the internal {@link org.apache.hadoop.metrics2.util.MetricSampleQuantiles} every interval. */
- private static class RolloverSample implements Runnable {
-
- MetricMutableQuantiles parent;
-
- public RolloverSample(MetricMutableQuantiles parent) {
- this.parent = parent;
- }
-
- @Override
- public void run() {
- synchronized (parent) {
- try {
- parent.previousCount = parent.estimator.getCount();
- parent.previousSnapshot = parent.estimator.snapshot();
- } catch (IOException e) {
- // Couldn't get a new snapshot because the window was empty
- parent.previousCount = 0;
- parent.previousSnapshot = null;
- }
- parent.estimator.clear();
- }
- parent.setChanged();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java
index f70413e..c381609 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.metrics2.MetricsExecutor;
/**
* Class to handle the ScheduledExecutorService{@link ScheduledExecutorService} used by
- * MetricMutableQuantiles{@link MetricMutableQuantiles}, MetricsRegionAggregateSourceImpl, and
+ * MetricsRegionAggregateSourceImpl, and
* JmxCacheBuster
*/
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java
new file mode 100644
index 0000000..3f6c4a7
--- /dev/null
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.lib;
+
+import org.apache.hadoop.hbase.util.Counter;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+
+public class MutableFastCounter extends MutableCounter {
+
+ private final Counter counter;
+
+ protected MutableFastCounter(MetricsInfo info, long iVal) {
+ super(info);
+ counter = new Counter(iVal);
+ }
+
+ @Override
+ public void incr() {
+ counter.increment();
+ setChanged();
+ }
+
+ /**
+ * Increment the value by a delta
+ * @param delta of the increment
+ */
+ public void incr(long delta) {
+ counter.add(delta);
+ setChanged();
+ }
+
+ @Override
+ public void snapshot(MetricsRecordBuilder builder, boolean all) {
+ if (all || changed()) {
+ builder.addCounter(info(), value());
+ clearChanged();
+ }
+ }
+
+ public long value() {
+ return counter.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java
index de29940..a6f0d14 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java
@@ -18,124 +18,81 @@
package org.apache.hadoop.metrics2.lib;
-import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Counter;
+import org.apache.hadoop.hbase.util.FastLongHistogram;
import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import com.yammer.metrics.stats.ExponentiallyDecayingSample;
-import com.yammer.metrics.stats.Sample;
-import com.yammer.metrics.stats.Snapshot;
-
/**
* A histogram implementation that runs in constant space, and exports to hadoop2's metrics2 system.
*/
@InterfaceAudience.Private
public class MutableHistogram extends MutableMetric implements MetricHistogram {
-
- private static final int DEFAULT_SAMPLE_SIZE = 2046;
- // the bias towards sampling from more recent data.
- // Per Cormode et al. an alpha of 0.015 strongly biases to the last 5 minutes
- private static final double DEFAULT_ALPHA = 0.015;
+ // Double buffer the two FastLongHistograms.
+ // As they are reset they learn how the buckets should be spaced
+ // So keep two around and use them
+ protected final FastLongHistogram histogram;
protected final String name;
protected final String desc;
- private final Sample sample;
- private final AtomicLong min;
- private final AtomicLong max;
- private final AtomicLong sum;
- private final AtomicLong count;
+
+ protected final Counter counter = new Counter(0);
public MutableHistogram(MetricsInfo info) {
this(info.name(), info.description());
}
public MutableHistogram(String name, String description) {
- this.name = StringUtils.capitalize(name);
- this.desc = StringUtils.uncapitalize(description);
- sample = new ExponentiallyDecayingSample(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA);
- count = new AtomicLong();
- min = new AtomicLong(Long.MAX_VALUE);
- max = new AtomicLong(Long.MIN_VALUE);
- sum = new AtomicLong();
- }
-
- public void add(final long val) {
- setChanged();
- count.incrementAndGet();
- sample.update(val);
- setMax(val);
- setMin(val);
- sum.getAndAdd(val);
+ this(name, description, Integer.MAX_VALUE << 2);
}
- private void setMax(final long potentialMax) {
- boolean done = false;
- while (!done) {
- final long currentMax = max.get();
- done = currentMax >= potentialMax
- || max.compareAndSet(currentMax, potentialMax);
- }
- }
-
- private void setMin(long potentialMin) {
- boolean done = false;
- while (!done) {
- final long currentMin = min.get();
- done = currentMin <= potentialMin
- || min.compareAndSet(currentMin, potentialMin);
- }
- }
-
- public long getMax() {
- if (count.get() > 0) {
- return max.get();
- }
- return 0L;
- }
+ protected MutableHistogram(String name, String description, long maxExpected) {
+ this.name = StringUtils.capitalize(name);
+ this.desc = StringUtils.uncapitalize(description);
- public long getMin() {
- if (count.get() > 0) {
- return min.get();
- }
- return 0L;
+ this.histogram = new FastLongHistogram(FastLongHistogram.DEFAULT_NBINS, 1, maxExpected);
}
- public double getMean() {
- long cCount = count.get();
- if (cCount > 0) {
- return sum.get() / (double) cCount;
- }
- return 0.0;
+ public void add(final long val) {
+ counter.increment();
+ histogram.add(val, 1);
}
@Override
- public void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) {
- if (all || changed()) {
- clearChanged();
- updateSnapshotMetrics(metricsRecordBuilder);
- }
+ public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) {
+ // Get a reference to the old histogram.
+ FastLongHistogram histo = histogram.reset();
+ updateSnapshotMetrics(metricsRecordBuilder, histo);
}
-
- public void updateSnapshotMetrics(MetricsRecordBuilder metricsRecordBuilder) {
- final Snapshot s = sample.getSnapshot();
- metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), count.get());
-
- metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), getMin());
- metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), getMax());
- metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), getMean());
- metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), s.getMedian());
- metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc),
- s.get75thPercentile());
- metricsRecordBuilder.addGauge(Interns.info(name + NINETIETH_PERCENTILE_METRIC_NAME, desc),
- s.getValue(0.90));
- metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc),
- s.get95thPercentile());
- metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc),
- s.get99thPercentile());
+ protected void updateSnapshotMetrics(MetricsRecordBuilder metricsRecordBuilder,
+ FastLongHistogram histo) {
+ metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), counter.get());
+ metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), histo.getMin());
+ metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), histo.getMax());
+ metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), histo.getMean());
+
+ long[] percentiles = histo.getQuantiles();
+
+ metricsRecordBuilder.addGauge(Interns.info(name + TWENTY_FIFTH_PERCENTILE_METRIC_NAME, desc),
+ percentiles[0]);
+ metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc),
+ percentiles[1]);
+ metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc),
+ percentiles[2]);
+ metricsRecordBuilder.addGauge(Interns.info(name + NINETIETH_PERCENTILE_METRIC_NAME, desc),
+ percentiles[3]);
+ metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc),
+ percentiles[4]);
+ metricsRecordBuilder.addGauge(Interns.info(name + NINETY_EIGHTH_PERCENTILE_METRIC_NAME, desc),
+ percentiles[5]);
+ metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc),
+ percentiles[6]);
+ metricsRecordBuilder.addGauge(
+ Interns.info(name + NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME, desc),
+ percentiles[7]);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java
index ac1f497..ac8aee0 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java
@@ -18,26 +18,29 @@
package org.apache.hadoop.metrics2.lib;
-import java.util.concurrent.atomic.AtomicLongArray;
-
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.FastLongHistogram;
+import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-
/**
* Extended histogram implementation with metric range counters.
*/
@InterfaceAudience.Private
-public abstract class MutableRangeHistogram extends MutableHistogram {
+public abstract class MutableRangeHistogram extends MutableHistogram implements MetricHistogram {
public MutableRangeHistogram(MetricsInfo info) {
this(info.name(), info.description());
}
public MutableRangeHistogram(String name, String description) {
- super(name, description);
+ this(name, description, Integer.MAX_VALUE << 2);
}
-
+
+ public MutableRangeHistogram(String name, String description, long expectedMax) {
+ super(name, description, expectedMax);
+ }
+
/**
* Returns the type of range histogram size or time
*/
@@ -46,49 +49,39 @@ public abstract class MutableRangeHistogram extends MutableHistogram {
/**
* Returns the ranges to be counted
*/
- public abstract long[] getRange();
-
- /**
- * Returns the range counts
- */
- public abstract AtomicLongArray getRangeVals();
+ public abstract long[] getRanges();
- @Override
- public void add(final long val) {
- super.add(val);
- updateBand(val);
- }
-
- private void updateBand(final long val) {
- int i;
- for (i=0; i<getRange().length && val > getRange()[i]; i++);
- getRangeVals().incrementAndGet(i);
- }
@Override
- public void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) {
- if (all || changed()) {
- clearChanged();
- updateSnapshotMetrics(metricsRecordBuilder);
- updateSnapshotRangeMetrics(metricsRecordBuilder);
- }
+ public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) {
+ // Get a reference to the old histogram.
+ FastLongHistogram histo = histogram.reset();
+ updateSnapshotMetrics(metricsRecordBuilder, histo);
+ updateSnapshotRangeMetrics(metricsRecordBuilder, histo);
}
-
- public void updateSnapshotRangeMetrics(MetricsRecordBuilder metricsRecordBuilder) {
- long prior = 0;
- for (int i = 0; i < getRange().length; i++) {
- long val = getRangeVals().get(i);
- if (val > 0) {
+
+ public void updateSnapshotRangeMetrics(MetricsRecordBuilder metricsRecordBuilder,
+ FastLongHistogram histogram) {
+ long priorRange = 0;
+ long cumNum = 0;
+
+ final long[] ranges = getRanges();
+ final String rangeType = getRangeType();
+ for (int i = 0; i < ranges.length - 1; i++) {
+ long val = histogram.getNumAtOrBelow(ranges[i]);
+ if (val - cumNum > 0) {
metricsRecordBuilder.addCounter(
- Interns.info(name + "_" + getRangeType() + "_" + prior + "-" + getRange()[i], desc), val);
+ Interns.info(name + "_" + rangeType + "_" + priorRange + "-" + ranges[i], desc),
+ val - cumNum);
}
- prior = getRange()[i];
+ priorRange = ranges[i];
+ cumNum = val;
}
- long val = getRangeVals().get(getRange().length);
- if (val > 0) {
+ long val = histogram.getCount();
+ if (val - cumNum > 0) {
metricsRecordBuilder.addCounter(
- Interns.info(name + "_" + getRangeType() + "_" + getRange()[getRange().length - 1] + "-inf", desc),
- getRangeVals().get(getRange().length));
+ Interns.info(name + "_" + rangeType + "_" + ranges[ranges.length - 1] + "-inf", desc),
+ val - cumNum);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3352173e/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java
index 2f1d57a..38e78a2 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.metrics2.lib;
-import java.util.concurrent.atomic.AtomicLongArray;
-
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.metrics2.MetricsInfo;
@@ -28,30 +26,29 @@ import org.apache.hadoop.metrics2.MetricsInfo;
*/
@InterfaceAudience.Private
public class MutableSizeHistogram extends MutableRangeHistogram {
- private final String rangeType = "SizeRangeCount";
- private final long[] ranges = {10,100,1000,10000,100000,1000000,10000000,100000000};
- private final AtomicLongArray rangeVals = new AtomicLongArray(getRange().length+1);
+ private final static String RANGE_TYPE = "SizeRangeCount";
+ private final static long[] RANGES = {10,100,1000,10000,100000,1000000,10000000,100000000};
public MutableSizeHistogram(MetricsInfo info) {
this(info.name(), info.description());
}
public MutableSizeHistogram(String name, String description) {
- super(name, description);
+ this(name, description, RANGES[RANGES.length-2]);
+ }
+
+ public MutableSizeHistogram(String name, String description, long expectedMax) {
+ super(name, description, expectedMax);
}
@Override
public String getRangeType() {
- return rangeType;
+ return RANGE_TYPE;
}
@Override
- public long[] getRange() {
- return ranges;
+ public long[] getRanges() {
+ return RANGES;
}
-
- @Override
- public AtomicLongArray getRangeVals() {
- return rangeVals;
- }
+
}