You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2017/08/14 15:53:36 UTC
[2/4] hbase git commit: HBASE-18469 Correct RegionServer metric of
totalRequestCount
HBASE-18469 Correct RegionServer metric of totalRequestCount
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c9f38bc7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c9f38bc7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c9f38bc7
Branch: refs/heads/branch-1.4
Commit: c9f38bc7f31057a723effd5b59d36760623cffb9
Parents: 2d06c1f7
Author: Yu Li <li...@apache.org>
Authored: Fri Aug 11 14:13:18 2017 +0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Aug 14 08:41:46 2017 -0700
----------------------------------------------------------------------
.../regionserver/MetricsRegionServerSource.java | 3 ++
.../MetricsRegionServerWrapper.java | 2 +
.../MetricsRegionServerSourceImpl.java | 2 +
.../MetricsRegionServerWrapperImpl.java | 5 +++
.../hbase/regionserver/RSRpcServices.java | 15 +++++--
.../MetricsRegionServerWrapperStub.java | 5 +++
.../regionserver/TestMetricsRegionServer.java | 4 ++
.../regionserver/TestRegionServerMetrics.java | 41 ++++++++++++++++++--
8 files changed, 71 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 06bdee6..3ac678e 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -244,6 +244,9 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String TOTAL_REQUEST_COUNT = "totalRequestCount";
String TOTAL_REQUEST_COUNT_DESC =
"Total number of requests this RegionServer has answered.";
+ String TOTAL_ROW_ACTION_REQUEST_COUNT = "totalRowActionRequestCount";
+ String TOTAL_ROW_ACTION_REQUEST_COUNT_DESC =
+ "Total number of region requests this RegionServer has answered, count by row-level action";
String READ_REQUEST_COUNT = "readRequestCount";
String READ_REQUEST_COUNT_DESC =
"Number of read requests this region server has answered.";
http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 7232063..7d7f66d 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -367,4 +367,6 @@ public interface MetricsRegionServerWrapper {
long getDeleteFamilyBloomHitCount();
long getTrailerHitCount();
+
+ long getTotalRowActionRequestCount();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 9b7f80a..cec122b 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -354,6 +354,8 @@ public class MetricsRegionServerSourceImpl
.addGauge(Interns.info(AVERAGE_REGION_SIZE, AVERAGE_REGION_SIZE_DESC), rsWrap.getAverageRegionSize())
.addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC),
rsWrap.getTotalRequestCount())
+ .addCounter(Interns.info(TOTAL_ROW_ACTION_REQUEST_COUNT, TOTAL_ROW_ACTION_REQUEST_COUNT_DESC),
+ rsWrap.getTotalRowActionRequestCount())
.addCounter(Interns.info(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC),
rsWrap.getReadRequestsCount())
.addCounter(Interns.info(WRITE_REQUEST_COUNT, WRITE_REQUEST_COUNT_DESC),
http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 35fd1ae..e805e75 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -174,6 +174,11 @@ class MetricsRegionServerWrapperImpl
}
@Override
+ public long getTotalRowActionRequestCount() {
+ return regionServer.rpcServices.requestRowActionCount.get();
+ }
+
+ @Override
public int getSplitQueueSize() {
if (this.regionServer.compactSplitThread == null) {
return 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 8317139..6afc3cc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -237,9 +237,13 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
*/
static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000;
- // Request counter. (Includes requests that are not serviced by regions.)
+ // Count only once for requests with multiple actions like multi/caching-scan/replayBatch
final Counter requestCount = new Counter();
+ // Request counter. (Excludes requests that are not serviced by regions.)
+ // Count rows for requests with multiple actions like multi/caching-scan/replayBatch
+ final Counter requestRowActionCount = new Counter();
+
// Request counter for rpc get
final Counter rpcGetRequestCount = new Counter();
@@ -990,7 +994,8 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
it.remove();
}
}
- requestCount.add(mutations.size());
+ requestCount.increment();
+ requestRowActionCount.add(mutations.size());
if (!region.getRegionInfo().isMetaTable()) {
regionServer.cacheFlusher.reclaimMemStoreMemory();
}
@@ -2193,6 +2198,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
try {
checkOpen();
requestCount.increment();
+ requestRowActionCount.increment();
rpcGetRequestCount.increment();
Region region = getRegion(request.getRegion());
@@ -2307,10 +2313,11 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
Boolean processed = null;
this.rpcMultiRequestCount.increment();
+ this.requestCount.increment();
Map<RegionSpecifier, ClientProtos.RegionLoadStats> regionStats = new HashMap<>(request
.getRegionActionCount());
for (RegionAction regionAction : request.getRegionActionList()) {
- this.requestCount.add(regionAction.getActionCount());
+ this.requestRowActionCount.add(regionAction.getActionCount());
OperationQuota quota;
Region region;
regionActionResultBuilder.clear();
@@ -2435,6 +2442,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
try {
checkOpen();
requestCount.increment();
+ requestRowActionCount.increment();
rpcMutateRequestCount.increment();
Region region = getRegion(request.getRegion());
MutateResponse.Builder builder = MutateResponse.newBuilder();
@@ -2872,6 +2880,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
}
}
region.updateReadRequestsCount(numOfResults);
+ requestRowActionCount.add(numOfResults);
long end = EnvironmentEdgeManager.currentTime();
long responseCellSize = context != null ? context.getResponseCellSize() : 0;
region.getMetrics().updateScanTime(end - before);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 32971a4..b573323 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -101,6 +101,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
}
@Override
+ public long getTotalRowActionRequestCount() {
+ return getReadRequestsCount() + getWriteRequestsCount();
+ }
+
+ @Override
public long getReadRequestsCount() {
return 997;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
index ff83c02..e1ef971 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
@@ -71,6 +71,10 @@ public class TestMetricsRegionServer {
HELPER.assertGauge("memstoreSize", 1025, serverSource);
HELPER.assertGauge("storeFileSize", 1900, serverSource);
HELPER.assertCounter("totalRequestCount", 899, serverSource);
+ HELPER.assertCounter("totalRowActionRequestCount",
+ HELPER.getCounter("readRequestCount", serverSource)
+ + HELPER.getCounter("writeRequestCount", serverSource),
+ serverSource);
HELPER.assertCounter("readRequestCount", 997, serverSource);
HELPER.assertCounter("writeRequestCount", 707, serverSource);
HELPER.assertCounter("checkMutateFailedCount", 401, serverSource);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c9f38bc7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 19041d0..ba12b9c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -124,6 +124,7 @@ public class TestRegionServerMetrics {
metricsRegionServer.getRegionServerWrapper().forceRecompute();
long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
+ long rowActionRequests = metricsHelper.getCounter("totalRowActionRequestCount", serverSource);
long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
@@ -133,6 +134,7 @@ public class TestRegionServerMetrics {
metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
+ metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 30, serverSource);
metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
@@ -143,6 +145,7 @@ public class TestRegionServerMetrics {
metricsRegionServer.getRegionServerWrapper().forceRecompute();
metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
+ metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 40, serverSource);
metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
@@ -170,27 +173,59 @@ public class TestRegionServerMetrics {
// By default, master doesn't host meta now.
// Adding some meta related requests
requests += 1;
+ rowActionRequests += 1;
readRequests ++;
metricsRegionServer.getRegionServerWrapper().forceRecompute();
- metricsHelper.assertCounter("totalRequestCount", requests + 50, serverSource);
+ metricsHelper.assertCounter("totalRequestCount", requests + 41, serverSource);
+ metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 50, serverSource);
metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
List<Put> puts = new ArrayList<>();
for (int i=0; i< 30; i++) {
- puts.add(p);
+ // put multiple lines for latter scan
+ Put put = new Put(Bytes.toBytes("" + i + "row")).addColumn(cfName, qualifier, initValue);
+ puts.add(put);
}
table.put(puts);
metricsRegionServer.getRegionServerWrapper().forceRecompute();
- metricsHelper.assertCounter("totalRequestCount", requests + 80, serverSource);
+ metricsHelper.assertCounter("totalRequestCount", requests + 42, serverSource);
+ metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 80, serverSource);
metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
+ doScan(table, 10, false);
+ metricsRegionServer.getRegionServerWrapper().forceRecompute();
+ metricsHelper.assertCounter("totalRequestCount", requests + 52, serverSource);
+ metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 90, serverSource);
+ metricsHelper.assertCounter("readRequestCount", readRequests + 30, serverSource);
+ metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
+
+ doScan(table, 10, true);
+ metricsRegionServer.getRegionServerWrapper().forceRecompute();
+ metricsHelper.assertCounter("totalRequestCount", requests + 53, serverSource);
+ metricsHelper.assertCounter("totalRowActionRequestCount", rowActionRequests + 100, serverSource);
+ metricsHelper.assertCounter("readRequestCount", readRequests + 40, serverSource);
+ metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
+
table.close();
}
+ public void doScan(Table table, int n, boolean caching) throws IOException {
+ Scan scan = new Scan();
+ if (caching) {
+ scan.setCaching(n);
+ } else {
+ scan.setCaching(1);
+ }
+ ResultScanner scanner = table.getScanner(scan);
+ for (int i = 0; i < n; i++) {
+ scanner.next();
+ }
+ }
+
@Test
public void testGet() throws Exception {
String tableNameString = "testGet";