You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/09/25 14:51:37 UTC
svn commit: r1389841 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/regionserver/metrics/
test/java/org/apache/hadoop/hbase/regionserver/
Author: mbautin
Date: Tue Sep 25 12:51:37 2012
New Revision: 1389841
URL: http://svn.apache.org/viewvc?rev=1389841&view=rev
Log:
[HBASE-6872] Number of records read/written per second
Author: adela
Summary: as in title
Test Plan: check NUM READS and NUM WRITES in testMultipleRegions() in TestRegionServerMetrics.java
Reviewers: liyintang
Reviewed By: liyintang
Differential Revision: https://phabricator.fb.com/D556908
Task ID: 1149883
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1389841&r1=1389840&r2=1389841&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Sep 25 12:51:37 2012
@@ -53,9 +53,7 @@ import java.util.concurrent.ExecutorServ
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -243,8 +241,8 @@ public class HRegionServer implements HR
// Leases
private Leases leases;
- // Request counter
- private volatile AtomicInteger requestCount = new AtomicInteger();
+ private volatile AtomicInteger numReads = new AtomicInteger();
+ private volatile AtomicInteger numWrites = new AtomicInteger();
// Info server. Default access so can be used by unit tests. REGIONSERVER
// is name of the webapp and the attribute name used stuffing this instance
@@ -639,7 +637,7 @@ public class HRegionServer implements HR
doMetrics();
MemoryUsage memory =
ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
- HServerLoad hsl = new HServerLoad(requestCount.get(),
+ HServerLoad hsl = new HServerLoad((numReads.get() + numWrites.get()),
(int)(memory.getUsed()/1024/1024),
(int)(memory.getMax()/1024/1024));
for (HRegion r: onlineRegions.values()) {
@@ -647,7 +645,8 @@ public class HRegionServer implements HR
}
// XXX add a field in serverInfo to report to fsOK to master?
this.serverInfo.setLoad(hsl);
- this.requestCount.set(0);
+ numReads.set(0);
+ numWrites.set(0);
addOutboundMsgs(outboundMessages);
HMsg msgs[] = this.hbaseMaster.regionServerReport(
serverInfo, outboundMessages.toArray(EMPTY_HMSG_ARRAY),
@@ -979,6 +978,8 @@ public class HRegionServer implements HR
this.dynamicMetrics = RegionServerDynamicMetrics.newInstance(this);
startServiceThreads();
isOnline = true;
+ this.numReads.set(0);
+ this.numWrites.set(0);
// Create the thread for the ThriftServer.
// NOTE this defaults to FALSE so you have to enable it in conf
@@ -1298,8 +1299,13 @@ public class HRegionServer implements HR
}
protected void metrics() {
+ int numReads = this.numReads.get();
+ int numWrites = this.numWrites.get();
+
this.metrics.regions.set(this.onlineRegions.size());
- this.metrics.incrementRequests(this.requestCount.get());
+ this.metrics.incrementRequests(numReads + numWrites);
+ this.metrics.numReads.inc(numReads);
+ this.metrics.numWrites.inc(numWrites);
// Is this too expensive every three seconds getting a lock on onlineRegions
// and then per store carried? Can I make metrics be sloppier and avoid
// the synchronizations?
@@ -1715,7 +1721,6 @@ public class HRegionServer implements HR
boolean znodeWritten = false;
while(!stopRequested.get()) {
try {
- this.requestCount.set(0);
MemoryUsage memory =
ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
HServerLoad hsl = new HServerLoad(0, (int)memory.getUsed()/1024/1024,
@@ -2232,7 +2237,7 @@ public class HRegionServer implements HR
@Override
public HRegionInfo getRegionInfo(final byte [] regionName)
throws NotServingRegionException {
- requestCount.incrementAndGet();
+ numReads.incrementAndGet();
return getRegion(regionName).getRegionInfo();
}
@@ -2242,12 +2247,11 @@ public class HRegionServer implements HR
final byte [] row, final byte [] family)
throws IOException {
checkOpen();
- requestCount.incrementAndGet();
+ numReads.incrementAndGet();
try {
// locate the region we're operating on
HRegion region = getRegion(regionName);
// ask the region for all the data
-
Result r = region.getClosestRowBefore(row, family);
return r;
} catch (Throwable t) {
@@ -2259,7 +2263,7 @@ public class HRegionServer implements HR
@Override
public Result get(byte [] regionName, Get get) throws IOException {
checkOpen();
- requestCount.incrementAndGet();
+ numReads.incrementAndGet();
try {
HRegion region = getRegion(regionName);
return region.get(get, getLockFromId(get.getLockId()));
@@ -2272,7 +2276,7 @@ public class HRegionServer implements HR
public Result[] get(byte[] regionName, List<Get> gets)
throws IOException {
checkOpen();
- requestCount.addAndGet(gets.size());
+ numReads.addAndGet(gets.size());
Result[] rets = new Result[gets.size()];
try {
HRegion region = getRegion(regionName);
@@ -2295,14 +2299,14 @@ public class HRegionServer implements HR
throw new IOException("Invalid arguments to atomicMutation " +
"regionName is null");
}
- requestCount.incrementAndGet();
+ numWrites.incrementAndGet();
try {
HRegion region = getRegion(regionName);
if (!region.getRegionInfo().isMetaTable()) {
this.cacheFlusher.reclaimMemStoreMemory();
}
for (RowMutations arm: armList) {
- this.requestCount.incrementAndGet();
+ numWrites.incrementAndGet();
region.mutateRow(arm);
}
} catch (Throwable t) {
@@ -2319,7 +2323,7 @@ public class HRegionServer implements HR
@Override
public boolean exists(byte [] regionName, Get get) throws IOException {
checkOpen();
- requestCount.incrementAndGet();
+ numReads.incrementAndGet();
try {
HRegion region = getRegion(regionName);
Result r = region.get(get, getLockFromId(get.getLockId()));
@@ -2336,7 +2340,7 @@ public class HRegionServer implements HR
throw new IllegalArgumentException("update has null row");
checkOpen();
- this.requestCount.incrementAndGet();
+ numWrites.incrementAndGet();
HRegion region = getRegion(regionName);
try {
if (!region.getRegionInfo().isMetaTable()) {
@@ -2376,7 +2380,7 @@ public class HRegionServer implements HR
opWithLocks[i++] = new Pair<Mutation, Integer>(p, lock);
}
- this.requestCount.addAndGet(mutations.size());
+ numWrites.addAndGet(mutations.size());
OperationStatusCode[] codes = region.batchMutateWithLocks(opWithLocks,
methodName);
for (i = 0; i < codes.length; i++) {
@@ -2393,7 +2397,7 @@ public class HRegionServer implements HR
final byte [] family, final byte [] qualifier, final byte [] value,
final Writable w, Integer lock) throws IOException {
checkOpen();
- this.requestCount.incrementAndGet();
+ numWrites.incrementAndGet();
HRegion region = getRegion(regionName);
try {
if (!region.getRegionInfo().isMetaTable()) {
@@ -2462,7 +2466,7 @@ public class HRegionServer implements HR
if (npe != null) {
throw new IOException("Invalid arguments to openScanner", npe);
}
- requestCount.incrementAndGet();
+ numReads.incrementAndGet();
try {
HRegion r = getRegion(regionName);
return addScanner(r.getScanner(scan));
@@ -2545,7 +2549,6 @@ public class HRegionServer implements HR
List<Result> results = new ArrayList<Result>(nbRows);
long currentScanResultSize = 0;
List<KeyValue> values = new ArrayList<KeyValue>();
-
int i = 0;
for (; i < nbRows && currentScanResultSize < maxScannerResultSize; i++) {
// Collect values to be returned here
@@ -2563,8 +2566,7 @@ public class HRegionServer implements HR
}
values.clear();
}
- requestCount.addAndGet(i);
-
+ numReads.addAndGet(i);
// Below is an ugly hack where we cast the InternalScanner to be a
// HRegion.RegionScanner. The alternative is to change InternalScanner
// interface but its used everywhere whereas we just need a bit of info
@@ -2586,7 +2588,7 @@ public class HRegionServer implements HR
public void close(final long scannerId) throws IOException {
try {
checkOpen();
- requestCount.incrementAndGet();
+ numReads.incrementAndGet();
String scannerName = String.valueOf(scannerId);
InternalScanner s = scanners.remove(scannerName);
if (s != null) {
@@ -2632,7 +2634,7 @@ public class HRegionServer implements HR
checkOpen();
try {
boolean writeToWAL = delete.getWriteToWAL();
- this.requestCount.incrementAndGet();
+ numWrites.incrementAndGet();
HRegion region = getRegion(regionName);
if (!region.getRegionInfo().isMetaTable()) {
this.cacheFlusher.reclaimMemStoreMemory();
@@ -2665,7 +2667,7 @@ public class HRegionServer implements HR
io.initCause(npe);
throw io;
}
- requestCount.incrementAndGet();
+ numReads.incrementAndGet();
try {
HRegion region = getRegion(regionName);
Integer r = region.obtainRowLock(row);
@@ -2724,7 +2726,7 @@ public class HRegionServer implements HR
io.initCause(npe);
throw io;
}
- requestCount.incrementAndGet();
+ numReads.incrementAndGet();
try {
HRegion region = getRegion(regionName);
String lockName = String.valueOf(lockId);
@@ -3001,9 +3003,14 @@ public class HRegionServer implements HR
return onlineRegions.get(Bytes.mapKey(regionName));
}
- /** @return the request count */
- public AtomicInteger getRequestCount() {
- return this.requestCount;
+ /** @return the number of reads */
+ public AtomicInteger getNumReads() {
+ return this.numReads;
+ }
+
+ /** @return the number of writes */
+ public AtomicInteger getNumWrites() {
+ return this.numWrites;
}
/** @return reference to FlushRequester */
@@ -3164,7 +3171,7 @@ public class HRegionServer implements HR
throw new IOException("Invalid arguments to incrementColumnValue " +
"regionName is null");
}
- requestCount.incrementAndGet();
+ numWrites.incrementAndGet();
try {
HRegion region = getRegion(regionName);
long retval = region.incrementColumnValue(row, family, qualifier, amount,
@@ -3514,5 +3521,4 @@ public class HRegionServer implements HR
}
return 0;
}
-
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1389841&r1=1389840&r2=1389841&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Tue Sep 25 12:51:37 2012
@@ -38,6 +38,8 @@ import org.apache.hadoop.metrics.jvm.Jvm
import org.apache.hadoop.metrics.util.MetricsIntValue;
import org.apache.hadoop.metrics.util.MetricsLongValue;
import org.apache.hadoop.metrics.util.MetricsRegistry;
+import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt;
+import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
import java.io.IOException;
@@ -200,6 +202,12 @@ public class RegionServerMetrics impleme
public final MetricsTimeVaryingRate mvccWaitTime =
new MetricsTimeVaryingRate("mvccWait", registry);
+ public final MetricsRate numReads =
+ new MetricsRate("numReads", registry);
+
+ public final MetricsRate numWrites =
+ new MetricsRate("numWrites", registry);
+
/**
* time each scheduled compaction takes
*/
@@ -286,6 +294,8 @@ public class RegionServerMetrics impleme
this.blockCacheEvictedMultiCount.pushMetric(this.metricsRecord);
this.blockCacheEvictedMemoryCount.pushMetric(this.metricsRecord);
this.blockCacheHitRatio.pushMetric(this.metricsRecord);
+ this.numReads.pushMetric(this.metricsRecord);
+ this.numWrites.pushMetric(this.metricsRecord);
// Be careful. Here is code for MTVR from up in hadoop:
// public synchronized void inc(final int numOps, final long time) {
@@ -422,6 +432,11 @@ public class RegionServerMetrics impleme
Integer.valueOf(this.memstoreSizeMB.get()));
sb = Strings.appendKeyValue(sb, "compactionQueueSize",
Integer.valueOf(this.compactionQueueSize.get()));
+ sb = Strings.appendKeyValue(sb, "numWrites",
+ Float.valueOf(this.numWrites.getPreviousIntervalValue()));
+ sb = Strings.appendKeyValue(sb, "numReads",
+ Float.valueOf(this.numReads.getPreviousIntervalValue()));
+
// Duplicate from jvmmetrics because metrics are private there so
// inaccessible.
MemoryUsage memory =
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java?rev=1389841&r1=1389840&r2=1389841&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java Tue Sep 25 12:51:37 2012
@@ -21,9 +21,9 @@ package org.apache.hadoop.hbase.regionse
import static org.junit.Assert.assertEquals;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -34,13 +34,12 @@ import org.apache.hadoop.hbase.client.HB
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.
StoreMetricType;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -207,4 +206,21 @@ public class TestRegionServerMetrics {
assertSizeMetric(tableName, cfs,
new int[] {kvLength, kvLength, kvLength, kvLength});
}
+
+ @Test
+ public void testNumReadsAndWrites() throws IOException, InterruptedException{
+ TEST_UTIL.createRandomTable(
+ "NumReadsWritesTest",
+ Arrays.asList(FAMILIES),
+ MAX_VERSIONS, NUM_COLS_PER_ROW, NUM_FLUSHES, NUM_REGIONS, 1000);
+ final HRegionServer rs =
+ TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+ rs.doMetrics();
+ for (HRegion r : rs.getOnlineRegions()) {
+ Get g = new Get(new byte[]{});
+ rs.get(r.getRegionName(), g);
+ }
+ Assert.assertEquals(rs.getOnlineRegions().size(), rs.getNumReads().get());
+ Assert.assertEquals(rs.getNumWrites().get(), 0);
+ }
}