You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nk...@apache.org on 2014/07/11 14:07:37 UTC
git commit: HBASE-11491 Add an option to sleep randomly during the
tests with the PE tool
Repository: hbase
Updated Branches:
refs/heads/master 77554df88 -> 4f3d5bbab
HBASE-11491 Add an option to sleep randomly during the tests with the PE tool
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4f3d5bba
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4f3d5bba
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4f3d5bba
Branch: refs/heads/master
Commit: 4f3d5bbab960ac7502fb0c6d9bdb8982b97333c3
Parents: 77554df
Author: Nicolas Liochon <nk...@gmail.com>
Authored: Fri Jul 11 14:06:28 2014 +0200
Committer: Nicolas Liochon <nk...@gmail.com>
Committed: Fri Jul 11 14:06:28 2014 +0200
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 42 ++++++++++++++------
1 file changed, 30 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4f3d5bba/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index d9fa0b7..6bc9fee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -254,9 +254,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
ObjectMapper mapper = new ObjectMapper();
TestOptions opts = mapper.readValue(value.toString(), TestOptions.class);
Configuration conf = HBaseConfiguration.create(context.getConfiguration());
+ final HConnection con = HConnectionManager.createConnection(conf);
// Evaluation task
- long elapsedTime = runOneClient(this.cmd, conf, opts, status);
+ long elapsedTime = runOneClient(this.cmd, conf, con, opts, status);
// Collect how much time the thing took. Report as map output and
// to the ELAPSED_TIME counter.
context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime);
@@ -377,6 +378,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
long[] timings = new long[opts.numClientThreads];
ExecutorService pool = Executors.newFixedThreadPool(opts.numClientThreads,
new ThreadFactoryBuilder().setNameFormat("TestClient-%s").build());
+ final HConnection con = HConnectionManager.createConnection(conf);
for (int i = 0; i < threads.length; i++) {
final int index = i;
threads[i] = pool.submit(new Callable<Long>() {
@@ -384,7 +386,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
public Long call() throws Exception {
TestOptions threadOpts = new TestOptions(opts);
if (threadOpts.startRow == 0) threadOpts.startRow = index * threadOpts.perClientRunRows;
- long elapsedTime = runOneClient(cmd, conf, threadOpts, new Status() {
+ long elapsedTime = runOneClient(cmd, conf, con, threadOpts, new Status() {
@Override
public void setStatus(final String msg) throws IOException {
LOG.info(msg);
@@ -397,6 +399,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
});
}
pool.shutdown();
+
for (int i = 0; i < threads.length; i++) {
try {
timings[i] = threads[i].get();
@@ -416,6 +419,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
+ "\tMin: " + timings[0] + "ms"
+ "\tMax: " + timings[timings.length - 1] + "ms"
+ "\tAvg: " + (total / timings.length) + "ms");
+
+ con.close();
+
return total;
}
@@ -556,6 +562,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
int noOfTags = 1;
boolean reportLatency = false;
int multiGet = 0;
+ int randomSleep = 0;
boolean inMemoryCF = false;
int presplitRegions = 0;
int replicas = HTableDescriptor.DEFAULT_REGION_REPLICATION;
@@ -599,6 +606,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.valueRandom = that.valueRandom;
this.valueSize = that.valueSize;
this.period = that.period;
+ this.randomSleep = that.randomSleep;
}
public boolean isNomapred() {
@@ -735,7 +743,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
private Histogram valueSize;
/**
- * Note that all subclasses of this class must provide a public contructor
+ * Note that all subclasses of this class must provide a public constructor
* that has the exact same list of arguments.
*/
Test(final HConnection con, final TestOptions options, final Status status) {
@@ -829,7 +837,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
* @return Elapsed time.
* @throws IOException
*/
- long test() throws IOException {
+ long test() throws IOException, InterruptedException {
testSetup();
LOG.info("Timed test starting in thread " + Thread.currentThread().getName());
final long startTime = System.nanoTime();
@@ -844,7 +852,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
/**
* Provides an extension point for tests that don't want a per row invocation.
*/
- void testTimed() throws IOException {
+ void testTimed() throws IOException, InterruptedException {
int lastRow = opts.startRow + opts.perClientRunRows;
// Report on completion of 1/10th of total.
for (int i = opts.startRow; i < lastRow; i++) {
@@ -925,7 +933,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
* Test for individual row.
* @param i Row index.
*/
- abstract void testRow(final int i) throws IOException;
+ abstract void testRow(final int i) throws IOException, InterruptedException;
}
@@ -1051,6 +1059,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
static class RandomReadTest extends Test {
private final Consistency consistency;
private ArrayList<Get> gets;
+ private Random rd = new Random();
RandomReadTest(HConnection con, TestOptions options, Status status) {
super(con, options, status);
@@ -1062,7 +1071,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
@Override
- void testRow(final int i) throws IOException {
+ void testRow(final int i) throws IOException, InterruptedException {
+ if (opts.randomSleep > 0) {
+ Thread.sleep(rd.nextInt(opts.randomSleep));
+ }
Get get = new Get(getRandomRow(this.rand, opts.totalRows));
get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
if (opts.filterAll) {
@@ -1320,15 +1332,14 @@ public class PerformanceEvaluation extends Configured implements Tool {
return format(random.nextInt(Integer.MAX_VALUE) % totalRows);
}
- static long runOneClient(final Class<? extends Test> cmd, Configuration conf, TestOptions opts,
- final Status status)
- throws IOException {
+ static long runOneClient(final Class<? extends Test> cmd, Configuration conf, HConnection con,
+ TestOptions opts, final Status status)
+ throws IOException, InterruptedException {
status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " +
opts.perClientRunRows + " rows");
long totalElapsedTime;
final Test t;
- HConnection con = HConnectionManager.createConnection(conf);
try {
Constructor<? extends Test> constructor =
cmd.getDeclaredConstructor(HConnection.class, TestOptions.class, Status.class);
@@ -1347,7 +1358,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
"ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" +
" (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime,
getAverageValueLength(opts)) + ")");
- con.close();
+
return totalElapsedTime;
}
@@ -1427,6 +1438,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
"by randomRead. Default: disabled");
System.err.println(" replicas Enable region replica testing. Defaults: 1.");
System.err.println(" splitPolicy Specify a custom RegionSplitPolicy for the table.");
+ System.err.println(" randomSleep Do a random sleep before each get between 0 and entered value. Defaults: 0");
System.err.println();
System.err.println(" Note: -D properties will be applied to the conf used. ");
System.err.println(" For example: ");
@@ -1597,6 +1609,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
continue;
}
+ final String randomSleep = "--randomSleep=";
+ if (cmd.startsWith(randomSleep)) {
+ opts.randomSleep = Integer.parseInt(cmd.substring(randomSleep.length()));
+ continue;
+ }
+
final String bloomFilter = "--bloomFilter";
if (cmd.startsWith(bloomFilter)) {
opts.bloomType = BloomType.valueOf(cmd.substring(bloomFilter.length()));