You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/05/05 01:38:53 UTC
[1/6] hbase git commit: HBASE-20513 Collect and emit ScanMetrics in
PerformanceEvaluation
Repository: hbase
Updated Branches:
refs/heads/branch-1 fd24083e2 -> 23b905408
refs/heads/branch-1.3 b6bb52110 -> 7a4a7d2e4
refs/heads/branch-1.4 489120afb -> f46a4ca69
refs/heads/branch-2 7df8e5e4a -> 991d78ca2
refs/heads/branch-2.0 3c4fadae4 -> 81f69e585
refs/heads/master 78ffd7ace -> 291dedbf8
HBASE-20513 Collect and emit ScanMetrics in PerformanceEvaluation
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/291dedbf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/291dedbf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/291dedbf
Branch: refs/heads/master
Commit: 291dedbf8185ede93808746e4233c2507d86f5cb
Parents: 78ffd7a
Author: Andrew Purtell <ap...@apache.org>
Authored: Tue May 1 10:58:09 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri May 4 17:59:12 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 119 ++++++++++++++++---
1 file changed, 100 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/291dedbf/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 25a1d3c..7fc064f 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterAllFilter;
@@ -1046,6 +1047,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
private String testName;
private Histogram latencyHistogram;
private Histogram valueSizeHistogram;
+ private Histogram rpcCallsHistogram;
+ private Histogram remoteRpcCallsHistogram;
+ private Histogram millisBetweenNextHistogram;
+ private Histogram regionsScannedHistogram;
+ private Histogram bytesInResultsHistogram;
+ private Histogram bytesInRemoteResultsHistogram;
private RandomDistribution.Zipf zipf;
/**
@@ -1102,6 +1109,34 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.valueSizeHistogram.update(valueSize);
}
+ void updateScanMetrics(final ScanMetrics metrics) {
+ Map<String,Long> metricsMap = metrics.getMetricsMap();
+ Long rpcCalls = metricsMap.get(ScanMetrics.RPC_CALLS_METRIC_NAME);
+ if (rpcCalls != null) {
+ this.rpcCallsHistogram.update(rpcCalls.longValue());
+ }
+ Long remoteRpcCalls = metricsMap.get(ScanMetrics.REMOTE_RPC_CALLS_METRIC_NAME);
+ if (remoteRpcCalls != null) {
+ this.remoteRpcCallsHistogram.update(remoteRpcCalls.longValue());
+ }
+ Long millisBetweenNext = metricsMap.get(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME);
+ if (millisBetweenNext != null) {
+ this.millisBetweenNextHistogram.update(millisBetweenNext.longValue());
+ }
+ Long regionsScanned = metricsMap.get(ScanMetrics.REGIONS_SCANNED_METRIC_NAME);
+ if (regionsScanned != null) {
+ this.regionsScannedHistogram.update(regionsScanned.longValue());
+ }
+ Long bytesInResults = metricsMap.get(ScanMetrics.BYTES_IN_RESULTS_METRIC_NAME);
+ if (bytesInResults != null && bytesInResults.longValue() > 0) {
+ this.bytesInResultsHistogram.update(bytesInResults.longValue());
+ }
+ Long bytesInRemoteResults = metricsMap.get(ScanMetrics.BYTES_IN_REMOTE_RESULTS_METRIC_NAME);
+ if (bytesInRemoteResults != null && bytesInRemoteResults.longValue() > 0) {
+ this.bytesInRemoteResultsHistogram.update(bytesInRemoteResults.longValue());
+ }
+ }
+
String generateStatus(final int sr, final int i, final int lr) {
return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +
(!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());
@@ -1123,10 +1158,19 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
void testSetup() throws IOException {
- createConnection();
- onStartup();
+ // test metrics
latencyHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ // scan metrics
+ rpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ remoteRpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ millisBetweenNextHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ regionsScannedHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ bytesInResultsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ bytesInRemoteResultsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+
+ createConnection();
+ onStartup();
}
abstract void createConnection() throws IOException;
@@ -1148,6 +1192,30 @@ public class PerformanceEvaluation extends Configured implements Tool {
+ YammerHistogramUtils.getHistogramReport(valueSizeHistogram));
status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.getCount());
status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram));
+ if (rpcCallsHistogram.getCount() > 0) {
+ status.setStatus("rpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(rpcCallsHistogram));
+ }
+ if (remoteRpcCallsHistogram.getCount() > 0) {
+ status.setStatus("remoteRpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(remoteRpcCallsHistogram));
+ }
+ if (millisBetweenNextHistogram.getCount() > 0) {
+ status.setStatus("millisBetweenNext (latency): " +
+ YammerHistogramUtils.getHistogramReport(millisBetweenNextHistogram));
+ }
+ if (regionsScannedHistogram.getCount() > 0) {
+ status.setStatus("regionsScanned (count): " +
+ YammerHistogramUtils.getHistogramReport(regionsScannedHistogram));
+ }
+ if (bytesInResultsHistogram.getCount() > 0) {
+ status.setStatus("bytesInResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInResultsHistogram));
+ }
+ if (bytesInRemoteResultsHistogram.getCount() > 0) {
+ status.setStatus("bytesInRemoteResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInRemoteResultsHistogram));
+ }
}
closeConnection();
receiverHost.closeReceivers();
@@ -1455,6 +1523,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void testTakedown() throws IOException {
if (this.testScanner != null) {
+ updateScanMetrics(this.testScanner.getScanMetrics());
this.testScanner.close();
}
super.testTakedown();
@@ -1466,7 +1535,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
Scan scan =
new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType);
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -1577,7 +1646,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
Scan scan = new Scan().withStartRow(getRandomRow(this.rand, opts.totalRows))
.setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks)
- .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType);
+ .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType)
+ .setScanMetricsEnabled(true);
FilterList list = new FilterList();
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
@@ -1593,10 +1663,14 @@ public class PerformanceEvaluation extends Configured implements Tool {
list.addFilter(new WhileMatchFilter(new PageFilter(120)));
scan.setFilter(list);
ResultScanner s = this.table.getScanner(scan);
- for (Result rr; (rr = s.next()) != null;) {
- updateValueSize(rr);
+ try {
+ for (Result rr; (rr = s.next()) != null;) {
+ updateValueSize(rr);
+ }
+ } finally {
+ updateScanMetrics(s.getScanMetrics());
+ s.close();
}
- s.close();
}
@Override
@@ -1618,7 +1692,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
Scan scan = new Scan().withStartRow(startAndStopRow.getFirst())
.withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType);
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
if (opts.filterAll) {
scan.setFilter(new FilterAllFilter());
}
@@ -1633,17 +1707,20 @@ public class PerformanceEvaluation extends Configured implements Tool {
Result r = null;
int count = 0;
ResultScanner s = this.table.getScanner(scan);
- for (; (r = s.next()) != null;) {
- updateValueSize(r);
- count++;
- }
- if (i % 100 == 0) {
- LOG.info(String.format("Scan for key range %s - %s returned %s rows",
+ try {
+ for (; (r = s.next()) != null;) {
+ updateValueSize(r);
+ count++;
+ }
+ if (i % 100 == 0) {
+ LOG.info(String.format("Scan for key range %s - %s returned %s rows",
Bytes.toString(startAndStopRow.getFirst()),
Bytes.toString(startAndStopRow.getSecond()), count));
+ }
+ } finally {
+ updateScanMetrics(s.getScanMetrics());
+ s.close();
}
-
- s.close();
}
protected abstract Pair<byte[],byte[]> getStartAndStopRow();
@@ -1824,7 +1901,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
if (this.testScanner == null) {
Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType);
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -2031,7 +2108,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
updateValueSize(r);
}
} finally {
- if (scanner != null) scanner.close();
+ if (scanner != null) {
+ updateScanMetrics(scanner.getScanMetrics());
+ scanner.close();
+ }
}
}
@@ -2046,7 +2126,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
list.addFilter(new FilterAllFilter());
}
Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks)
- .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType);
+ .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType)
+ .setScanMetricsEnabled(true);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
[2/6] hbase git commit: HBASE-20513 Collect and emit ScanMetrics in
PerformanceEvaluation
Posted by ap...@apache.org.
HBASE-20513 Collect and emit ScanMetrics in PerformanceEvaluation
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/81f69e58
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/81f69e58
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/81f69e58
Branch: refs/heads/branch-2.0
Commit: 81f69e585159840b622b9030e4eb6ebf35a7e6ae
Parents: 3c4fada
Author: Andrew Purtell <ap...@apache.org>
Authored: Tue May 1 10:58:09 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri May 4 17:59:16 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 119 ++++++++++++++++---
1 file changed, 100 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/81f69e58/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index a0d5572..11905aa 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterAllFilter;
@@ -1046,6 +1047,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
private String testName;
private Histogram latencyHistogram;
private Histogram valueSizeHistogram;
+ private Histogram rpcCallsHistogram;
+ private Histogram remoteRpcCallsHistogram;
+ private Histogram millisBetweenNextHistogram;
+ private Histogram regionsScannedHistogram;
+ private Histogram bytesInResultsHistogram;
+ private Histogram bytesInRemoteResultsHistogram;
private RandomDistribution.Zipf zipf;
/**
@@ -1102,6 +1109,34 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.valueSizeHistogram.update(valueSize);
}
+ void updateScanMetrics(final ScanMetrics metrics) {
+ Map<String,Long> metricsMap = metrics.getMetricsMap();
+ Long rpcCalls = metricsMap.get(ScanMetrics.RPC_CALLS_METRIC_NAME);
+ if (rpcCalls != null) {
+ this.rpcCallsHistogram.update(rpcCalls.longValue());
+ }
+ Long remoteRpcCalls = metricsMap.get(ScanMetrics.REMOTE_RPC_CALLS_METRIC_NAME);
+ if (remoteRpcCalls != null) {
+ this.remoteRpcCallsHistogram.update(remoteRpcCalls.longValue());
+ }
+ Long millisBetweenNext = metricsMap.get(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME);
+ if (millisBetweenNext != null) {
+ this.millisBetweenNextHistogram.update(millisBetweenNext.longValue());
+ }
+ Long regionsScanned = metricsMap.get(ScanMetrics.REGIONS_SCANNED_METRIC_NAME);
+ if (regionsScanned != null) {
+ this.regionsScannedHistogram.update(regionsScanned.longValue());
+ }
+ Long bytesInResults = metricsMap.get(ScanMetrics.BYTES_IN_RESULTS_METRIC_NAME);
+ if (bytesInResults != null && bytesInResults.longValue() > 0) {
+ this.bytesInResultsHistogram.update(bytesInResults.longValue());
+ }
+ Long bytesInRemoteResults = metricsMap.get(ScanMetrics.BYTES_IN_REMOTE_RESULTS_METRIC_NAME);
+ if (bytesInRemoteResults != null && bytesInRemoteResults.longValue() > 0) {
+ this.bytesInRemoteResultsHistogram.update(bytesInRemoteResults.longValue());
+ }
+ }
+
String generateStatus(final int sr, final int i, final int lr) {
return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +
(!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());
@@ -1123,10 +1158,19 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
void testSetup() throws IOException {
- createConnection();
- onStartup();
+ // test metrics
latencyHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ // scan metrics
+ rpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ remoteRpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ millisBetweenNextHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ regionsScannedHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ bytesInResultsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ bytesInRemoteResultsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+
+ createConnection();
+ onStartup();
}
abstract void createConnection() throws IOException;
@@ -1148,6 +1192,30 @@ public class PerformanceEvaluation extends Configured implements Tool {
+ YammerHistogramUtils.getHistogramReport(valueSizeHistogram));
status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.getCount());
status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram));
+ if (rpcCallsHistogram.getCount() > 0) {
+ status.setStatus("rpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(rpcCallsHistogram));
+ }
+ if (remoteRpcCallsHistogram.getCount() > 0) {
+ status.setStatus("remoteRpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(remoteRpcCallsHistogram));
+ }
+ if (millisBetweenNextHistogram.getCount() > 0) {
+ status.setStatus("millisBetweenNext (latency): " +
+ YammerHistogramUtils.getHistogramReport(millisBetweenNextHistogram));
+ }
+ if (regionsScannedHistogram.getCount() > 0) {
+ status.setStatus("regionsScanned (count): " +
+ YammerHistogramUtils.getHistogramReport(regionsScannedHistogram));
+ }
+ if (bytesInResultsHistogram.getCount() > 0) {
+ status.setStatus("bytesInResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInResultsHistogram));
+ }
+ if (bytesInRemoteResultsHistogram.getCount() > 0) {
+ status.setStatus("bytesInRemoteResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInRemoteResultsHistogram));
+ }
}
closeConnection();
receiverHost.closeReceivers();
@@ -1455,6 +1523,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void testTakedown() throws IOException {
if (this.testScanner != null) {
+ updateScanMetrics(this.testScanner.getScanMetrics());
this.testScanner.close();
}
super.testTakedown();
@@ -1466,7 +1535,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
Scan scan =
new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType);
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -1577,7 +1646,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
Scan scan = new Scan().withStartRow(getRandomRow(this.rand, opts.totalRows))
.setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks)
- .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType);
+ .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType)
+ .setScanMetricsEnabled(true);
FilterList list = new FilterList();
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
@@ -1593,10 +1663,14 @@ public class PerformanceEvaluation extends Configured implements Tool {
list.addFilter(new WhileMatchFilter(new PageFilter(120)));
scan.setFilter(list);
ResultScanner s = this.table.getScanner(scan);
- for (Result rr; (rr = s.next()) != null;) {
- updateValueSize(rr);
+ try {
+ for (Result rr; (rr = s.next()) != null;) {
+ updateValueSize(rr);
+ }
+ } finally {
+ updateScanMetrics(s.getScanMetrics());
+ s.close();
}
- s.close();
}
@Override
@@ -1618,7 +1692,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
Scan scan = new Scan().withStartRow(startAndStopRow.getFirst())
.withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType);
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
if (opts.filterAll) {
scan.setFilter(new FilterAllFilter());
}
@@ -1633,17 +1707,20 @@ public class PerformanceEvaluation extends Configured implements Tool {
Result r = null;
int count = 0;
ResultScanner s = this.table.getScanner(scan);
- for (; (r = s.next()) != null;) {
- updateValueSize(r);
- count++;
- }
- if (i % 100 == 0) {
- LOG.info(String.format("Scan for key range %s - %s returned %s rows",
+ try {
+ for (; (r = s.next()) != null;) {
+ updateValueSize(r);
+ count++;
+ }
+ if (i % 100 == 0) {
+ LOG.info(String.format("Scan for key range %s - %s returned %s rows",
Bytes.toString(startAndStopRow.getFirst()),
Bytes.toString(startAndStopRow.getSecond()), count));
+ }
+ } finally {
+ updateScanMetrics(s.getScanMetrics());
+ s.close();
}
-
- s.close();
}
protected abstract Pair<byte[],byte[]> getStartAndStopRow();
@@ -1824,7 +1901,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
if (this.testScanner == null) {
Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType);
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -2031,7 +2108,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
updateValueSize(r);
}
} finally {
- if (scanner != null) scanner.close();
+ if (scanner != null) {
+ updateScanMetrics(scanner.getScanMetrics());
+ scanner.close();
+ }
}
}
@@ -2046,7 +2126,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
list.addFilter(new FilterAllFilter());
}
Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks)
- .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType);
+ .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType)
+ .setScanMetricsEnabled(true);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
[6/6] hbase git commit: HBASE-20513 Collect and emit ScanMetrics in
PerformanceEvaluation
Posted by ap...@apache.org.
HBASE-20513 Collect and emit ScanMetrics in PerformanceEvaluation
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7a4a7d2e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7a4a7d2e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7a4a7d2e
Branch: refs/heads/branch-1.3
Commit: 7a4a7d2e4ab581083229e62228eea3b5916b32df
Parents: b6bb521
Author: Andrew Purtell <ap...@apache.org>
Authored: Tue May 1 10:58:09 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri May 4 18:00:01 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 127 ++++++++++++++++---
1 file changed, 108 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7a4a7d2e/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index faf52d0..f2097d4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
@@ -959,6 +960,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
private String testName;
private Histogram latencyHistogram;
private Histogram valueSizeHistogram;
+ private Histogram rpcCallsHistogram;
+ private Histogram remoteRpcCallsHistogram;
+ private Histogram millisBetweenNextHistogram;
+ private Histogram regionsScannedHistogram;
+ private Histogram bytesInResultsHistogram;
+ private Histogram bytesInRemoteResultsHistogram;
private RandomDistribution.Zipf zipf;
/**
@@ -1012,6 +1019,34 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.valueSizeHistogram.update(valueSize);
}
+ void updateScanMetrics(final ScanMetrics metrics) {
+ Map<String,Long> metricsMap = metrics.getMetricsMap();
+ Long rpcCalls = metricsMap.get(ScanMetrics.RPC_CALLS_METRIC_NAME);
+ if (rpcCalls != null) {
+ this.rpcCallsHistogram.update(rpcCalls.longValue());
+ }
+ Long remoteRpcCalls = metricsMap.get(ScanMetrics.REMOTE_RPC_CALLS_METRIC_NAME);
+ if (remoteRpcCalls != null) {
+ this.remoteRpcCallsHistogram.update(remoteRpcCalls.longValue());
+ }
+ Long millisBetweenNext = metricsMap.get(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME);
+ if (millisBetweenNext != null) {
+ this.millisBetweenNextHistogram.update(millisBetweenNext.longValue());
+ }
+ Long regionsScanned = metricsMap.get(ScanMetrics.REGIONS_SCANNED_METRIC_NAME);
+ if (regionsScanned != null) {
+ this.regionsScannedHistogram.update(regionsScanned.longValue());
+ }
+ Long bytesInResults = metricsMap.get(ScanMetrics.BYTES_IN_RESULTS_METRIC_NAME);
+ if (bytesInResults != null && bytesInResults.longValue() > 0) {
+ this.bytesInResultsHistogram.update(bytesInResults.longValue());
+ }
+ Long bytesInRemoteResults = metricsMap.get(ScanMetrics.BYTES_IN_REMOTE_RESULTS_METRIC_NAME);
+ if (bytesInRemoteResults != null && bytesInRemoteResults.longValue() > 0) {
+ this.bytesInRemoteResultsHistogram.update(bytesInRemoteResults.longValue());
+ }
+ }
+
String generateStatus(final int sr, final int i, final int lr) {
return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +
(!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());
@@ -1033,12 +1068,22 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
void testSetup() throws IOException {
+ // test metrics
+ latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ // scan metrics
+ rpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ remoteRpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ millisBetweenNextHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ regionsScannedHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ bytesInResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ bytesInRemoteResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+
if (!opts.oneCon) {
this.connection = ConnectionFactory.createConnection(conf);
}
+
onStartup();
- latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
- valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
}
abstract void onStartup() throws IOException;
@@ -1058,6 +1103,30 @@ public class PerformanceEvaluation extends Configured implements Tool {
+ YammerHistogramUtils.getHistogramReport(valueSizeHistogram));
status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.count());
status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram));
+ if (rpcCallsHistogram.count() > 0) {
+ status.setStatus("rpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(rpcCallsHistogram));
+ }
+ if (remoteRpcCallsHistogram.count() > 0) {
+ status.setStatus("remoteRpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(remoteRpcCallsHistogram));
+ }
+ if (millisBetweenNextHistogram.count() > 0) {
+ status.setStatus("millisBetweenNext (latency): " +
+ YammerHistogramUtils.getHistogramReport(millisBetweenNextHistogram));
+ }
+ if (regionsScannedHistogram.count() > 0) {
+ status.setStatus("regionsScanned (count): " +
+ YammerHistogramUtils.getHistogramReport(regionsScannedHistogram));
+ }
+ if (bytesInResultsHistogram.count() > 0) {
+ status.setStatus("bytesInResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInResultsHistogram));
+ }
+ if (bytesInRemoteResultsHistogram.count() > 0) {
+ status.setStatus("bytesInRemoteResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInRemoteResultsHistogram));
+ }
}
if (!opts.oneCon) {
connection.close();
@@ -1185,7 +1254,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void testRow(final int i) throws IOException {
Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows));
- scan.setCaching(opts.caching);
FilterList list = new FilterList();
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
@@ -1200,11 +1268,19 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
list.addFilter(new WhileMatchFilter(new PageFilter(120)));
scan.setFilter(list);
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
ResultScanner s = this.table.getScanner(scan);
- for (Result rr; (rr = s.next()) != null;) {
- updateValueSize(rr);
+ try {
+ for (Result rr; (rr = s.next()) != null;) {
+ updateValueSize(rr);
+ }
+ } finally {
+ if (s != null) {
+ updateScanMetrics(scan.getScanMetrics());
+ s.close();
+ }
}
- s.close();
}
@Override
@@ -1224,7 +1300,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
- scan.setCaching(opts.caching);
if (opts.filterAll) {
scan.setFilter(new FilterAllFilter());
}
@@ -1236,20 +1311,27 @@ public class PerformanceEvaluation extends Configured implements Tool {
} else {
scan.addFamily(FAMILY_NAME);
}
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
Result r = null;
int count = 0;
ResultScanner s = this.table.getScanner(scan);
- for (; (r = s.next()) != null;) {
- updateValueSize(r);
- count++;
- }
- if (i % 100 == 0) {
- LOG.info(String.format("Scan for key range %s - %s returned %s rows",
+ try {
+ for (; (r = s.next()) != null;) {
+ updateValueSize(r);
+ count++;
+ }
+ if (i % 100 == 0) {
+ LOG.info(String.format("Scan for key range %s - %s returned %s rows",
Bytes.toString(startAndStopRow.getFirst()),
Bytes.toString(startAndStopRow.getSecond()), count));
+ }
+ } finally {
+ if (s != null) {
+ updateScanMetrics(scan.getScanMetrics());
+ s.close();
+ }
}
-
- s.close();
}
protected abstract Pair<byte[],byte[]> getStartAndStopRow();
@@ -1409,6 +1491,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class ScanTest extends TableTest {
+ private Scan scan;
private ResultScanner testScanner;
ScanTest(Connection con, TestOptions options, Status status) {
@@ -1418,6 +1501,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void testTakedown() throws IOException {
if (this.testScanner != null) {
+ updateScanMetrics(scan.getScanMetrics());
this.testScanner.close();
}
super.testTakedown();
@@ -1428,7 +1512,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
if (this.testScanner == null) {
Scan scan = new Scan(format(opts.startRow));
- scan.setCaching(opts.caching);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -1440,7 +1523,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
if (opts.filterAll) {
scan.setFilter(new FilterAllFilter());
}
- this.testScanner = table.getScanner(scan);
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
+ this.testScanner = table.getScanner(scan);
}
Result r = testScanner.next();
updateValueSize(r);
@@ -1632,7 +1717,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
updateValueSize(r);
}
} finally {
- if (scanner != null) scanner.close();
+ if (scanner != null) {
+ updateScanMetrics(scan.getScanMetrics());
+ scanner.close();
+ }
}
}
@@ -1647,7 +1735,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
list.addFilter(new FilterAllFilter());
}
Scan scan = new Scan();
- scan.setCaching(opts.caching);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -1657,6 +1744,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
scan.addFamily(FAMILY_NAME);
}
scan.setFilter(list);
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
return scan;
}
}
[5/6] hbase git commit: HBASE-20513 Collect and emit ScanMetrics in
PerformanceEvaluation
Posted by ap...@apache.org.
HBASE-20513 Collect and emit ScanMetrics in PerformanceEvaluation
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f46a4ca6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f46a4ca6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f46a4ca6
Branch: refs/heads/branch-1.4
Commit: f46a4ca69a44b1f9f017132d7b4a9c39d615cfc9
Parents: 489120a
Author: Andrew Purtell <ap...@apache.org>
Authored: Tue May 1 10:58:09 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri May 4 17:59:57 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 127 ++++++++++++++++---
1 file changed, 108 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f46a4ca6/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 60d6cb6..d31e39f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
@@ -973,6 +974,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
private String testName;
private Histogram latencyHistogram;
private Histogram valueSizeHistogram;
+ private Histogram rpcCallsHistogram;
+ private Histogram remoteRpcCallsHistogram;
+ private Histogram millisBetweenNextHistogram;
+ private Histogram regionsScannedHistogram;
+ private Histogram bytesInResultsHistogram;
+ private Histogram bytesInRemoteResultsHistogram;
private RandomDistribution.Zipf zipf;
/**
@@ -1030,6 +1037,34 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.valueSizeHistogram.update(valueSize);
}
+ void updateScanMetrics(final ScanMetrics metrics) {
+ Map<String,Long> metricsMap = metrics.getMetricsMap();
+ Long rpcCalls = metricsMap.get(ScanMetrics.RPC_CALLS_METRIC_NAME);
+ if (rpcCalls != null) {
+ this.rpcCallsHistogram.update(rpcCalls.longValue());
+ }
+ Long remoteRpcCalls = metricsMap.get(ScanMetrics.REMOTE_RPC_CALLS_METRIC_NAME);
+ if (remoteRpcCalls != null) {
+ this.remoteRpcCallsHistogram.update(remoteRpcCalls.longValue());
+ }
+ Long millisBetweenNext = metricsMap.get(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME);
+ if (millisBetweenNext != null) {
+ this.millisBetweenNextHistogram.update(millisBetweenNext.longValue());
+ }
+ Long regionsScanned = metricsMap.get(ScanMetrics.REGIONS_SCANNED_METRIC_NAME);
+ if (regionsScanned != null) {
+ this.regionsScannedHistogram.update(regionsScanned.longValue());
+ }
+ Long bytesInResults = metricsMap.get(ScanMetrics.BYTES_IN_RESULTS_METRIC_NAME);
+ if (bytesInResults != null && bytesInResults.longValue() > 0) {
+ this.bytesInResultsHistogram.update(bytesInResults.longValue());
+ }
+ Long bytesInRemoteResults = metricsMap.get(ScanMetrics.BYTES_IN_REMOTE_RESULTS_METRIC_NAME);
+ if (bytesInRemoteResults != null && bytesInRemoteResults.longValue() > 0) {
+ this.bytesInRemoteResultsHistogram.update(bytesInRemoteResults.longValue());
+ }
+ }
+
String generateStatus(final int sr, final int i, final int lr) {
return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +
(!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());
@@ -1051,12 +1086,22 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
void testSetup() throws IOException {
+ // test metrics
+ latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ // scan metrics
+ rpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ remoteRpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ millisBetweenNextHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ regionsScannedHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ bytesInResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ bytesInRemoteResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+
if (!opts.oneCon) {
this.connection = ConnectionFactory.createConnection(conf);
}
+
onStartup();
- latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
- valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
}
abstract void onStartup() throws IOException;
@@ -1076,6 +1121,30 @@ public class PerformanceEvaluation extends Configured implements Tool {
+ YammerHistogramUtils.getHistogramReport(valueSizeHistogram));
status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.count());
status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram));
+ if (rpcCallsHistogram.count() > 0) {
+ status.setStatus("rpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(rpcCallsHistogram));
+ }
+ if (remoteRpcCallsHistogram.count() > 0) {
+ status.setStatus("remoteRpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(remoteRpcCallsHistogram));
+ }
+ if (millisBetweenNextHistogram.count() > 0) {
+ status.setStatus("millisBetweenNext (latency): " +
+ YammerHistogramUtils.getHistogramReport(millisBetweenNextHistogram));
+ }
+ if (regionsScannedHistogram.count() > 0) {
+ status.setStatus("regionsScanned (count): " +
+ YammerHistogramUtils.getHistogramReport(regionsScannedHistogram));
+ }
+ if (bytesInResultsHistogram.count() > 0) {
+ status.setStatus("bytesInResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInResultsHistogram));
+ }
+ if (bytesInRemoteResultsHistogram.count() > 0) {
+ status.setStatus("bytesInRemoteResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInRemoteResultsHistogram));
+ }
}
if (!opts.oneCon) {
connection.close();
@@ -1203,7 +1272,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void testRow(final int i) throws IOException {
Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows));
- scan.setCaching(opts.caching);
FilterList list = new FilterList();
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
@@ -1218,11 +1286,19 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
list.addFilter(new WhileMatchFilter(new PageFilter(120)));
scan.setFilter(list);
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
ResultScanner s = this.table.getScanner(scan);
- for (Result rr; (rr = s.next()) != null;) {
- updateValueSize(rr);
+ try {
+ for (Result rr; (rr = s.next()) != null;) {
+ updateValueSize(rr);
+ }
+ } finally {
+ if (s != null) {
+ updateScanMetrics(scan.getScanMetrics());
+ s.close();
+ }
}
- s.close();
}
@Override
@@ -1242,7 +1318,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
- scan.setCaching(opts.caching);
if (opts.filterAll) {
scan.setFilter(new FilterAllFilter());
}
@@ -1254,20 +1329,27 @@ public class PerformanceEvaluation extends Configured implements Tool {
} else {
scan.addFamily(FAMILY_NAME);
}
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
Result r = null;
int count = 0;
ResultScanner s = this.table.getScanner(scan);
- for (; (r = s.next()) != null;) {
- updateValueSize(r);
- count++;
- }
- if (i % 100 == 0) {
- LOG.info(String.format("Scan for key range %s - %s returned %s rows",
+ try {
+ for (; (r = s.next()) != null;) {
+ updateValueSize(r);
+ count++;
+ }
+ if (i % 100 == 0) {
+ LOG.info(String.format("Scan for key range %s - %s returned %s rows",
Bytes.toString(startAndStopRow.getFirst()),
Bytes.toString(startAndStopRow.getSecond()), count));
+ }
+ } finally {
+ if (s != null) {
+ updateScanMetrics(scan.getScanMetrics());
+ s.close();
+ }
}
-
- s.close();
}
protected abstract Pair<byte[],byte[]> getStartAndStopRow();
@@ -1427,6 +1509,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class ScanTest extends TableTest {
+ private Scan scan;
private ResultScanner testScanner;
ScanTest(Connection con, TestOptions options, Status status) {
@@ -1436,6 +1519,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void testTakedown() throws IOException {
if (this.testScanner != null) {
+ updateScanMetrics(scan.getScanMetrics());
this.testScanner.close();
}
super.testTakedown();
@@ -1446,7 +1530,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
if (this.testScanner == null) {
Scan scan = new Scan(format(opts.startRow));
- scan.setCaching(opts.caching);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -1458,7 +1541,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
if (opts.filterAll) {
scan.setFilter(new FilterAllFilter());
}
- this.testScanner = table.getScanner(scan);
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
+ this.testScanner = table.getScanner(scan);
}
Result r = testScanner.next();
updateValueSize(r);
@@ -1650,7 +1735,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
updateValueSize(r);
}
} finally {
- if (scanner != null) scanner.close();
+ if (scanner != null) {
+ updateScanMetrics(scan.getScanMetrics());
+ scanner.close();
+ }
}
}
@@ -1665,7 +1753,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
list.addFilter(new FilterAllFilter());
}
Scan scan = new Scan();
- scan.setCaching(opts.caching);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -1675,6 +1762,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
scan.addFamily(FAMILY_NAME);
}
scan.setFilter(list);
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
return scan;
}
}
[4/6] hbase git commit: HBASE-20513 Collect and emit ScanMetrics in
PerformanceEvaluation
Posted by ap...@apache.org.
HBASE-20513 Collect and emit ScanMetrics in PerformanceEvaluation
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/23b90540
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/23b90540
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/23b90540
Branch: refs/heads/branch-1
Commit: 23b9054089b2e529a45db63ee100e4748ce10f2f
Parents: fd24083
Author: Andrew Purtell <ap...@apache.org>
Authored: Tue May 1 10:58:09 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri May 4 17:59:28 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 127 ++++++++++++++++---
1 file changed, 108 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/23b90540/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 60d6cb6..d31e39f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
@@ -973,6 +974,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
private String testName;
private Histogram latencyHistogram;
private Histogram valueSizeHistogram;
+ private Histogram rpcCallsHistogram;
+ private Histogram remoteRpcCallsHistogram;
+ private Histogram millisBetweenNextHistogram;
+ private Histogram regionsScannedHistogram;
+ private Histogram bytesInResultsHistogram;
+ private Histogram bytesInRemoteResultsHistogram;
private RandomDistribution.Zipf zipf;
/**
@@ -1030,6 +1037,34 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.valueSizeHistogram.update(valueSize);
}
+ void updateScanMetrics(final ScanMetrics metrics) {
+ Map<String,Long> metricsMap = metrics.getMetricsMap();
+ Long rpcCalls = metricsMap.get(ScanMetrics.RPC_CALLS_METRIC_NAME);
+ if (rpcCalls != null) {
+ this.rpcCallsHistogram.update(rpcCalls.longValue());
+ }
+ Long remoteRpcCalls = metricsMap.get(ScanMetrics.REMOTE_RPC_CALLS_METRIC_NAME);
+ if (remoteRpcCalls != null) {
+ this.remoteRpcCallsHistogram.update(remoteRpcCalls.longValue());
+ }
+ Long millisBetweenNext = metricsMap.get(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME);
+ if (millisBetweenNext != null) {
+ this.millisBetweenNextHistogram.update(millisBetweenNext.longValue());
+ }
+ Long regionsScanned = metricsMap.get(ScanMetrics.REGIONS_SCANNED_METRIC_NAME);
+ if (regionsScanned != null) {
+ this.regionsScannedHistogram.update(regionsScanned.longValue());
+ }
+ Long bytesInResults = metricsMap.get(ScanMetrics.BYTES_IN_RESULTS_METRIC_NAME);
+ if (bytesInResults != null && bytesInResults.longValue() > 0) {
+ this.bytesInResultsHistogram.update(bytesInResults.longValue());
+ }
+ Long bytesInRemoteResults = metricsMap.get(ScanMetrics.BYTES_IN_REMOTE_RESULTS_METRIC_NAME);
+ if (bytesInRemoteResults != null && bytesInRemoteResults.longValue() > 0) {
+ this.bytesInRemoteResultsHistogram.update(bytesInRemoteResults.longValue());
+ }
+ }
+
String generateStatus(final int sr, final int i, final int lr) {
return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +
(!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());
@@ -1051,12 +1086,22 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
void testSetup() throws IOException {
+ // test metrics
+ latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ // scan metrics
+ rpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ remoteRpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ millisBetweenNextHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ regionsScannedHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ bytesInResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+ bytesInRemoteResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
+
if (!opts.oneCon) {
this.connection = ConnectionFactory.createConnection(conf);
}
+
onStartup();
- latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
- valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
}
abstract void onStartup() throws IOException;
@@ -1076,6 +1121,30 @@ public class PerformanceEvaluation extends Configured implements Tool {
+ YammerHistogramUtils.getHistogramReport(valueSizeHistogram));
status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.count());
status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram));
+ if (rpcCallsHistogram.count() > 0) {
+ status.setStatus("rpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(rpcCallsHistogram));
+ }
+ if (remoteRpcCallsHistogram.count() > 0) {
+ status.setStatus("remoteRpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(remoteRpcCallsHistogram));
+ }
+ if (millisBetweenNextHistogram.count() > 0) {
+ status.setStatus("millisBetweenNext (latency): " +
+ YammerHistogramUtils.getHistogramReport(millisBetweenNextHistogram));
+ }
+ if (regionsScannedHistogram.count() > 0) {
+ status.setStatus("regionsScanned (count): " +
+ YammerHistogramUtils.getHistogramReport(regionsScannedHistogram));
+ }
+ if (bytesInResultsHistogram.count() > 0) {
+ status.setStatus("bytesInResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInResultsHistogram));
+ }
+ if (bytesInRemoteResultsHistogram.count() > 0) {
+ status.setStatus("bytesInRemoteResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInRemoteResultsHistogram));
+ }
}
if (!opts.oneCon) {
connection.close();
@@ -1203,7 +1272,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void testRow(final int i) throws IOException {
Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows));
- scan.setCaching(opts.caching);
FilterList list = new FilterList();
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
@@ -1218,11 +1286,19 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
list.addFilter(new WhileMatchFilter(new PageFilter(120)));
scan.setFilter(list);
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
ResultScanner s = this.table.getScanner(scan);
- for (Result rr; (rr = s.next()) != null;) {
- updateValueSize(rr);
+ try {
+ for (Result rr; (rr = s.next()) != null;) {
+ updateValueSize(rr);
+ }
+ } finally {
+ if (s != null) {
+ updateScanMetrics(scan.getScanMetrics());
+ s.close();
+ }
}
- s.close();
}
@Override
@@ -1242,7 +1318,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
- scan.setCaching(opts.caching);
if (opts.filterAll) {
scan.setFilter(new FilterAllFilter());
}
@@ -1254,20 +1329,27 @@ public class PerformanceEvaluation extends Configured implements Tool {
} else {
scan.addFamily(FAMILY_NAME);
}
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
Result r = null;
int count = 0;
ResultScanner s = this.table.getScanner(scan);
- for (; (r = s.next()) != null;) {
- updateValueSize(r);
- count++;
- }
- if (i % 100 == 0) {
- LOG.info(String.format("Scan for key range %s - %s returned %s rows",
+ try {
+ for (; (r = s.next()) != null;) {
+ updateValueSize(r);
+ count++;
+ }
+ if (i % 100 == 0) {
+ LOG.info(String.format("Scan for key range %s - %s returned %s rows",
Bytes.toString(startAndStopRow.getFirst()),
Bytes.toString(startAndStopRow.getSecond()), count));
+ }
+ } finally {
+ if (s != null) {
+ updateScanMetrics(scan.getScanMetrics());
+ s.close();
+ }
}
-
- s.close();
}
protected abstract Pair<byte[],byte[]> getStartAndStopRow();
@@ -1427,6 +1509,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class ScanTest extends TableTest {
+ private Scan scan;
private ResultScanner testScanner;
ScanTest(Connection con, TestOptions options, Status status) {
@@ -1436,6 +1519,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void testTakedown() throws IOException {
if (this.testScanner != null) {
+ updateScanMetrics(scan.getScanMetrics());
this.testScanner.close();
}
super.testTakedown();
@@ -1446,7 +1530,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
if (this.testScanner == null) {
Scan scan = new Scan(format(opts.startRow));
- scan.setCaching(opts.caching);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -1458,7 +1541,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
if (opts.filterAll) {
scan.setFilter(new FilterAllFilter());
}
- this.testScanner = table.getScanner(scan);
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
+ this.testScanner = table.getScanner(scan);
}
Result r = testScanner.next();
updateValueSize(r);
@@ -1650,7 +1735,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
updateValueSize(r);
}
} finally {
- if (scanner != null) scanner.close();
+ if (scanner != null) {
+ updateScanMetrics(scan.getScanMetrics());
+ scanner.close();
+ }
}
}
@@ -1665,7 +1753,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
list.addFilter(new FilterAllFilter());
}
Scan scan = new Scan();
- scan.setCaching(opts.caching);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -1675,6 +1762,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
scan.addFamily(FAMILY_NAME);
}
scan.setFilter(list);
+ scan.setCaching(opts.caching);
+ scan.setScanMetricsEnabled(true);
return scan;
}
}
[3/6] hbase git commit: HBASE-20513 Collect and emit ScanMetrics in
PerformanceEvaluation
Posted by ap...@apache.org.
HBASE-20513 Collect and emit ScanMetrics in PerformanceEvaluation
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/991d78ca
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/991d78ca
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/991d78ca
Branch: refs/heads/branch-2
Commit: 991d78ca2bf838a24e504590b1124367d255ec03
Parents: 7df8e5e
Author: Andrew Purtell <ap...@apache.org>
Authored: Tue May 1 10:58:09 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri May 4 17:59:20 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 119 ++++++++++++++++---
1 file changed, 100 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/991d78ca/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index a0d5572..11905aa 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterAllFilter;
@@ -1046,6 +1047,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
private String testName;
private Histogram latencyHistogram;
private Histogram valueSizeHistogram;
+ private Histogram rpcCallsHistogram;
+ private Histogram remoteRpcCallsHistogram;
+ private Histogram millisBetweenNextHistogram;
+ private Histogram regionsScannedHistogram;
+ private Histogram bytesInResultsHistogram;
+ private Histogram bytesInRemoteResultsHistogram;
private RandomDistribution.Zipf zipf;
/**
@@ -1102,6 +1109,34 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.valueSizeHistogram.update(valueSize);
}
+ void updateScanMetrics(final ScanMetrics metrics) {
+ Map<String,Long> metricsMap = metrics.getMetricsMap();
+ Long rpcCalls = metricsMap.get(ScanMetrics.RPC_CALLS_METRIC_NAME);
+ if (rpcCalls != null) {
+ this.rpcCallsHistogram.update(rpcCalls.longValue());
+ }
+ Long remoteRpcCalls = metricsMap.get(ScanMetrics.REMOTE_RPC_CALLS_METRIC_NAME);
+ if (remoteRpcCalls != null) {
+ this.remoteRpcCallsHistogram.update(remoteRpcCalls.longValue());
+ }
+ Long millisBetweenNext = metricsMap.get(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME);
+ if (millisBetweenNext != null) {
+ this.millisBetweenNextHistogram.update(millisBetweenNext.longValue());
+ }
+ Long regionsScanned = metricsMap.get(ScanMetrics.REGIONS_SCANNED_METRIC_NAME);
+ if (regionsScanned != null) {
+ this.regionsScannedHistogram.update(regionsScanned.longValue());
+ }
+ Long bytesInResults = metricsMap.get(ScanMetrics.BYTES_IN_RESULTS_METRIC_NAME);
+ if (bytesInResults != null && bytesInResults.longValue() > 0) {
+ this.bytesInResultsHistogram.update(bytesInResults.longValue());
+ }
+ Long bytesInRemoteResults = metricsMap.get(ScanMetrics.BYTES_IN_REMOTE_RESULTS_METRIC_NAME);
+ if (bytesInRemoteResults != null && bytesInRemoteResults.longValue() > 0) {
+ this.bytesInRemoteResultsHistogram.update(bytesInRemoteResults.longValue());
+ }
+ }
+
String generateStatus(final int sr, final int i, final int lr) {
return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +
(!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());
@@ -1123,10 +1158,19 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
void testSetup() throws IOException {
- createConnection();
- onStartup();
+ // test metrics
latencyHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ // scan metrics
+ rpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ remoteRpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ millisBetweenNextHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ regionsScannedHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ bytesInResultsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+ bytesInRemoteResultsHistogram = YammerHistogramUtils.newHistogram(new UniformReservoir(1024 * 500));
+
+ createConnection();
+ onStartup();
}
abstract void createConnection() throws IOException;
@@ -1148,6 +1192,30 @@ public class PerformanceEvaluation extends Configured implements Tool {
+ YammerHistogramUtils.getHistogramReport(valueSizeHistogram));
status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.getCount());
status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram));
+ if (rpcCallsHistogram.getCount() > 0) {
+ status.setStatus("rpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(rpcCallsHistogram));
+ }
+ if (remoteRpcCallsHistogram.getCount() > 0) {
+ status.setStatus("remoteRpcCalls (count): " +
+ YammerHistogramUtils.getHistogramReport(remoteRpcCallsHistogram));
+ }
+ if (millisBetweenNextHistogram.getCount() > 0) {
+ status.setStatus("millisBetweenNext (latency): " +
+ YammerHistogramUtils.getHistogramReport(millisBetweenNextHistogram));
+ }
+ if (regionsScannedHistogram.getCount() > 0) {
+ status.setStatus("regionsScanned (count): " +
+ YammerHistogramUtils.getHistogramReport(regionsScannedHistogram));
+ }
+ if (bytesInResultsHistogram.getCount() > 0) {
+ status.setStatus("bytesInResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInResultsHistogram));
+ }
+ if (bytesInRemoteResultsHistogram.getCount() > 0) {
+ status.setStatus("bytesInRemoteResults (size): " +
+ YammerHistogramUtils.getHistogramReport(bytesInRemoteResultsHistogram));
+ }
}
closeConnection();
receiverHost.closeReceivers();
@@ -1455,6 +1523,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void testTakedown() throws IOException {
if (this.testScanner != null) {
+ updateScanMetrics(this.testScanner.getScanMetrics());
this.testScanner.close();
}
super.testTakedown();
@@ -1466,7 +1535,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
Scan scan =
new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType);
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -1577,7 +1646,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
Scan scan = new Scan().withStartRow(getRandomRow(this.rand, opts.totalRows))
.setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks)
- .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType);
+ .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType)
+ .setScanMetricsEnabled(true);
FilterList list = new FilterList();
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
@@ -1593,10 +1663,14 @@ public class PerformanceEvaluation extends Configured implements Tool {
list.addFilter(new WhileMatchFilter(new PageFilter(120)));
scan.setFilter(list);
ResultScanner s = this.table.getScanner(scan);
- for (Result rr; (rr = s.next()) != null;) {
- updateValueSize(rr);
+ try {
+ for (Result rr; (rr = s.next()) != null;) {
+ updateValueSize(rr);
+ }
+ } finally {
+ updateScanMetrics(s.getScanMetrics());
+ s.close();
}
- s.close();
}
@Override
@@ -1618,7 +1692,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
Scan scan = new Scan().withStartRow(startAndStopRow.getFirst())
.withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType);
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
if (opts.filterAll) {
scan.setFilter(new FilterAllFilter());
}
@@ -1633,17 +1707,20 @@ public class PerformanceEvaluation extends Configured implements Tool {
Result r = null;
int count = 0;
ResultScanner s = this.table.getScanner(scan);
- for (; (r = s.next()) != null;) {
- updateValueSize(r);
- count++;
- }
- if (i % 100 == 0) {
- LOG.info(String.format("Scan for key range %s - %s returned %s rows",
+ try {
+ for (; (r = s.next()) != null;) {
+ updateValueSize(r);
+ count++;
+ }
+ if (i % 100 == 0) {
+ LOG.info(String.format("Scan for key range %s - %s returned %s rows",
Bytes.toString(startAndStopRow.getFirst()),
Bytes.toString(startAndStopRow.getSecond()), count));
+ }
+ } finally {
+ updateScanMetrics(s.getScanMetrics());
+ s.close();
}
-
- s.close();
}
protected abstract Pair<byte[],byte[]> getStartAndStopRow();
@@ -1824,7 +1901,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
if (this.testScanner == null) {
Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
.setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
- .setReadType(opts.scanReadType);
+ .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
@@ -2031,7 +2108,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
updateValueSize(r);
}
} finally {
- if (scanner != null) scanner.close();
+ if (scanner != null) {
+ updateScanMetrics(scanner.getScanMetrics());
+ scanner.close();
+ }
}
}
@@ -2046,7 +2126,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
list.addFilter(new FilterAllFilter());
}
Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks)
- .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType);
+ .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType)
+ .setScanMetricsEnabled(true);
if (opts.addColumns) {
for (int column = 0; column < opts.columns; column++) {
byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);