You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nk...@apache.org on 2014/06/24 11:34:43 UTC
git commit: HBASE-11390 PerformanceEvaluation: add an option to use a
single connection
Repository: hbase
Updated Branches:
refs/heads/master b2914be9d -> e991fd805
HBASE-11390 PerformanceEvaluation: add an option to use a single connection
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e991fd80
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e991fd80
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e991fd80
Branch: refs/heads/master
Commit: e991fd8055112e2ddc0bbeade2ef74504e2407dd
Parents: b2914be
Author: Nicolas Liochon <nk...@gmail.com>
Authored: Tue Jun 24 11:34:45 2014 +0200
Committer: Nicolas Liochon <nk...@gmail.com>
Committed: Tue Jun 24 11:34:45 2014 +0200
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 83 ++++++++++++--------
1 file changed, 50 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e991fd80/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index d54cfa1..5190fb1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
@@ -503,6 +504,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.flushCommits = that.flushCommits;
this.writeToWAL = that.writeToWAL;
this.autoFlush = that.autoFlush;
+ this.oneCon = that.oneCon;
this.useTags = that.useTags;
this.noOfTags = that.noOfTags;
this.reportLatency = that.reportLatency;
@@ -531,6 +533,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
public boolean flushCommits = true;
public boolean writeToWAL = true;
public boolean autoFlush = false;
+ public boolean oneCon = false;
public boolean useTags = false;
public int noOfTags = 1;
public boolean reportLatency = false;
@@ -577,8 +580,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
* Note that all subclasses of this class must provide a public contructor
* that has the exact same list of arguments.
*/
- Test(final Configuration conf, final TestOptions options, final Status status) {
- this.conf = conf;
+ Test(final HConnection con, final TestOptions options, final Status status) {
+ this.connection = con;
+ this.conf = con.getConfiguration();
this.opts = options;
this.status = status;
this.testName = this.getClass().getSimpleName();
@@ -631,9 +635,11 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
void testSetup() throws IOException {
- this.connection = HConnectionManager.createConnection(conf);
- this.table = connection.getTable(opts.tableName);
- this.table.setAutoFlush(opts.autoFlush, true);
+ if (!opts.oneCon) {
+ this.connection = HConnectionManager.createConnection(conf);
+ }
+ this.table = new HTable(TableName.valueOf(opts.tableName), connection);
+ this.table.setAutoFlushTo(opts.autoFlush);
try {
Constructor<?> ctor =
@@ -654,7 +660,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.table.flushCommits();
}
table.close();
- connection.close();
+ if (!opts.oneCon) {
+ connection.close();
+ }
receiverHost.closeReceivers();
}
@@ -765,8 +773,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
@SuppressWarnings("unused")
static class RandomSeekScanTest extends Test {
- RandomSeekScanTest(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ RandomSeekScanTest(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -795,8 +803,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static abstract class RandomScanWithRangeTest extends Test {
- RandomScanWithRangeTest(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ RandomScanWithRangeTest(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -811,7 +819,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
int count = 0;
ResultScanner s = this.table.getScanner(scan);
for (; (r = s.next()) != null;) {
- updateValueSize(r);;
+ updateValueSize(r);
count++;
}
if (i % 100 == 0) {
@@ -839,8 +847,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class RandomScanWithRange10Test extends RandomScanWithRangeTest {
- RandomScanWithRange10Test(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ RandomScanWithRange10Test(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -850,8 +858,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class RandomScanWithRange100Test extends RandomScanWithRangeTest {
- RandomScanWithRange100Test(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ RandomScanWithRange100Test(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -861,8 +869,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class RandomScanWithRange1000Test extends RandomScanWithRangeTest {
- RandomScanWithRange1000Test(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ RandomScanWithRange1000Test(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -872,8 +880,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class RandomScanWithRange10000Test extends RandomScanWithRangeTest {
- RandomScanWithRange10000Test(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ RandomScanWithRange10000Test(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -885,8 +893,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
static class RandomReadTest extends Test {
private ArrayList<Get> gets;
- RandomReadTest(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ RandomReadTest(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
if (opts.multiGet > 0) {
LOG.info("MultiGet enabled. Sending GETs in batches of " + opts.multiGet + ".");
this.gets = new ArrayList<Get>(opts.multiGet);
@@ -930,8 +938,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class RandomWriteTest extends Test {
- RandomWriteTest(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ RandomWriteTest(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -962,8 +970,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
static class ScanTest extends Test {
private ResultScanner testScanner;
- ScanTest(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ ScanTest(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -993,8 +1001,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class SequentialReadTest extends Test {
- SequentialReadTest(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ SequentialReadTest(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -1009,8 +1017,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
static class SequentialWriteTest extends Test {
- SequentialWriteTest(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ SequentialWriteTest(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -1041,8 +1049,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
static class FilteredScanTest extends Test {
protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName());
- FilteredScanTest(Configuration conf, TestOptions options, Status status) {
- super(conf, options, status);
+ FilteredScanTest(HConnection con, TestOptions options, Status status) {
+ super(con, options, status);
}
@Override
@@ -1159,10 +1167,11 @@ public class PerformanceEvaluation extends Configured implements Tool {
long totalElapsedTime;
final Test t;
+ HConnection con = HConnectionManager.createConnection(conf);
try {
Constructor<? extends Test> constructor =
- cmd.getDeclaredConstructor(Configuration.class, TestOptions.class, Status.class);
- t = constructor.newInstance(conf, opts, status);
+ cmd.getDeclaredConstructor(HConnection.class, TestOptions.class, Status.class);
+ t = constructor.newInstance(con, opts, status);
} catch (NoSuchMethodException e) {
throw new IllegalArgumentException("Invalid command class: " +
cmd.getName() + ". It does not provide a constructor as described by " +
@@ -1177,6 +1186,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
"ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" +
" (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime,
getAverageValueLength(opts)) + ")");
+ con.close();
return totalElapsedTime;
}
@@ -1229,6 +1239,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
"Default: false");
System.err.println(" writeToWAL Set writeToWAL on puts. Default: True");
System.err.println(" autoFlush Set autoFlush on htable. Default: False");
+ System.err.println(" oneCon all the threads share the same connection. Default: False");
System.err.println(" presplit Create presplit table. Recommended for accurate perf " +
"analysis (see guide). Default: disabled");
System.err.println(" inmemory Tries to keep the HFiles of the CF " +
@@ -1366,6 +1377,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
continue;
}
+ final String onceCon = "--oneCon=";
+ if (cmd.startsWith(onceCon)) {
+ opts.oneCon = Boolean.parseBoolean(cmd.substring(onceCon.length()));
+ continue;
+ }
+
final String presplit = "--presplit=";
if (cmd.startsWith(presplit)) {
opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length()));