You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nk...@apache.org on 2014/07/11 14:31:54 UTC
[1/2] git commit: HBASE-11491 Add an option to sleep randomly during
the tests with the PE tool
Repository: hbase
Updated Branches:
refs/heads/branch-1 3cdbe2ae5 -> c80b45104
HBASE-11491 Add an option to sleep randomly during the tests with the PE tool
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a74122a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a74122a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a74122a
Branch: refs/heads/branch-1
Commit: 9a74122a5c3aacd4c7f107bd8027c4a28fc2f561
Parents: 3cdbe2a
Author: Nicolas Liochon <nk...@gmail.com>
Authored: Fri Jul 11 14:06:28 2014 +0200
Committer: Nicolas Liochon <nk...@gmail.com>
Committed: Fri Jul 11 14:29:56 2014 +0200
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 42 ++++++++++++++------
1 file changed, 30 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9a74122a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index d9fa0b7..6bc9fee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -254,9 +254,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
ObjectMapper mapper = new ObjectMapper();
TestOptions opts = mapper.readValue(value.toString(), TestOptions.class);
Configuration conf = HBaseConfiguration.create(context.getConfiguration());
+ final HConnection con = HConnectionManager.createConnection(conf);
// Evaluation task
- long elapsedTime = runOneClient(this.cmd, conf, opts, status);
+ long elapsedTime = runOneClient(this.cmd, conf, con, opts, status);
// Collect how much time the thing took. Report as map output and
// to the ELAPSED_TIME counter.
context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime);
@@ -377,6 +378,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
long[] timings = new long[opts.numClientThreads];
ExecutorService pool = Executors.newFixedThreadPool(opts.numClientThreads,
new ThreadFactoryBuilder().setNameFormat("TestClient-%s").build());
+ final HConnection con = HConnectionManager.createConnection(conf);
for (int i = 0; i < threads.length; i++) {
final int index = i;
threads[i] = pool.submit(new Callable<Long>() {
@@ -384,7 +386,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
public Long call() throws Exception {
TestOptions threadOpts = new TestOptions(opts);
if (threadOpts.startRow == 0) threadOpts.startRow = index * threadOpts.perClientRunRows;
- long elapsedTime = runOneClient(cmd, conf, threadOpts, new Status() {
+ long elapsedTime = runOneClient(cmd, conf, con, threadOpts, new Status() {
@Override
public void setStatus(final String msg) throws IOException {
LOG.info(msg);
@@ -397,6 +399,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
});
}
pool.shutdown();
+
for (int i = 0; i < threads.length; i++) {
try {
timings[i] = threads[i].get();
@@ -416,6 +419,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
+ "\tMin: " + timings[0] + "ms"
+ "\tMax: " + timings[timings.length - 1] + "ms"
+ "\tAvg: " + (total / timings.length) + "ms");
+
+ con.close();
+
return total;
}
@@ -556,6 +562,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
int noOfTags = 1;
boolean reportLatency = false;
int multiGet = 0;
+ int randomSleep = 0;
boolean inMemoryCF = false;
int presplitRegions = 0;
int replicas = HTableDescriptor.DEFAULT_REGION_REPLICATION;
@@ -599,6 +606,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.valueRandom = that.valueRandom;
this.valueSize = that.valueSize;
this.period = that.period;
+ this.randomSleep = that.randomSleep;
}
public boolean isNomapred() {
@@ -735,7 +743,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
private Histogram valueSize;
/**
- * Note that all subclasses of this class must provide a public contructor
+ * Note that all subclasses of this class must provide a public constructor
* that has the exact same list of arguments.
*/
Test(final HConnection con, final TestOptions options, final Status status) {
@@ -829,7 +837,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
* @return Elapsed time.
* @throws IOException
*/
- long test() throws IOException {
+ long test() throws IOException, InterruptedException {
testSetup();
LOG.info("Timed test starting in thread " + Thread.currentThread().getName());
final long startTime = System.nanoTime();
@@ -844,7 +852,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
/**
* Provides an extension point for tests that don't want a per row invocation.
*/
- void testTimed() throws IOException {
+ void testTimed() throws IOException, InterruptedException {
int lastRow = opts.startRow + opts.perClientRunRows;
// Report on completion of 1/10th of total.
for (int i = opts.startRow; i < lastRow; i++) {
@@ -925,7 +933,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
* Test for individual row.
* @param i Row index.
*/
- abstract void testRow(final int i) throws IOException;
+ abstract void testRow(final int i) throws IOException, InterruptedException;
}
@@ -1051,6 +1059,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
static class RandomReadTest extends Test {
private final Consistency consistency;
private ArrayList<Get> gets;
+ private Random rd = new Random();
RandomReadTest(HConnection con, TestOptions options, Status status) {
super(con, options, status);
@@ -1062,7 +1071,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
@Override
- void testRow(final int i) throws IOException {
+ void testRow(final int i) throws IOException, InterruptedException {
+ if (opts.randomSleep > 0) {
+ Thread.sleep(rd.nextInt(opts.randomSleep));
+ }
Get get = new Get(getRandomRow(this.rand, opts.totalRows));
get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
if (opts.filterAll) {
@@ -1320,15 +1332,14 @@ public class PerformanceEvaluation extends Configured implements Tool {
return format(random.nextInt(Integer.MAX_VALUE) % totalRows);
}
- static long runOneClient(final Class<? extends Test> cmd, Configuration conf, TestOptions opts,
- final Status status)
- throws IOException {
+ static long runOneClient(final Class<? extends Test> cmd, Configuration conf, HConnection con,
+ TestOptions opts, final Status status)
+ throws IOException, InterruptedException {
status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " +
opts.perClientRunRows + " rows");
long totalElapsedTime;
final Test t;
- HConnection con = HConnectionManager.createConnection(conf);
try {
Constructor<? extends Test> constructor =
cmd.getDeclaredConstructor(HConnection.class, TestOptions.class, Status.class);
@@ -1347,7 +1358,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
"ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" +
" (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime,
getAverageValueLength(opts)) + ")");
- con.close();
+
return totalElapsedTime;
}
@@ -1427,6 +1438,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
"by randomRead. Default: disabled");
System.err.println(" replicas Enable region replica testing. Defaults: 1.");
System.err.println(" splitPolicy Specify a custom RegionSplitPolicy for the table.");
+ System.err.println(" randomSleep Do a random sleep before each get between 0 and entered value. Defaults: 0");
System.err.println();
System.err.println(" Note: -D properties will be applied to the conf used. ");
System.err.println(" For example: ");
@@ -1597,6 +1609,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
continue;
}
+ final String randomSleep = "--randomSleep=";
+ if (cmd.startsWith(randomSleep)) {
+ opts.randomSleep = Integer.parseInt(cmd.substring(randomSleep.length()));
+ continue;
+ }
+
final String bloomFilter = "--bloomFilter";
if (cmd.startsWith(bloomFilter)) {
opts.bloomType = BloomType.valueOf(cmd.substring(bloomFilter.length()));
[2/2] git commit: HBASE-11492 The servers do not honor the tcpNoDelay
option
Posted by nk...@apache.org.
HBASE-11492 The servers do not honor the tcpNoDelay option
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c80b4510
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c80b4510
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c80b4510
Branch: refs/heads/branch-1
Commit: c80b4510443552d13f21f5eda62f94daff318565
Parents: 9a74122
Author: Nicolas Liochon <nk...@gmail.com>
Authored: Fri Jul 11 14:15:08 2014 +0200
Committer: Nicolas Liochon <nk...@gmail.com>
Committed: Fri Jul 11 14:30:10 2014 +0200
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c80b4510/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 0cb002f..d6be4a8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -30,6 +30,7 @@ import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketException;
+import java.net.StandardSocketOptions;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.channels.CancelledKeyException;
@@ -752,7 +753,7 @@ public class RpcServer implements RpcServerInterface {
while ((channel = server.accept()) != null) {
try {
channel.configureBlocking(false);
- channel.socket().setTcpNoDelay(tcpNoDelay);
+ channel.setOption(StandardSocketOptions.TCP_NODELAY, tcpNoDelay);
channel.socket().setKeepAlive(tcpKeepAlive);
} catch (IOException ioe) {
channel.close();