You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/05/08 12:16:43 UTC
[02/31] hbase git commit: HBASE-20523 PE tool should support
configuring client side buffering sizes (Ram)
HBASE-20523 PE tool should support configuring client side buffering sizes
(Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8e6ff689
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8e6ff689
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8e6ff689
Branch: refs/heads/HBASE-19064
Commit: 8e6ff689e806138e602a46730886d8c5f524fdcd
Parents: 5e14e12
Author: Vasudevan <ra...@intel.com>
Authored: Mon May 7 12:50:24 2018 +0530
Committer: Vasudevan <ra...@intel.com>
Committed: Mon May 7 12:57:20 2018 +0530
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 23 +++++++++++++++++++-
.../hadoop/hbase/TestPerformanceEvaluation.java | 10 +++++++++
2 files changed, 32 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8e6ff689/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 7fc064f..42dc984 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.AsyncConnection;
import org.apache.hadoop.hbase.client.AsyncTable;
import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.BufferedMutatorParams;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Consistency;
@@ -666,6 +667,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
boolean asyncPrefetch = false;
boolean cacheBlocks = true;
Scan.ReadType scanReadType = Scan.ReadType.DEFAULT;
+ long bufferSize = 2l * 1024l * 1024l;
public TestOptions() {}
@@ -715,6 +717,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.asyncPrefetch = that.asyncPrefetch;
this.cacheBlocks = that.cacheBlocks;
this.scanReadType = that.scanReadType;
+ this.bufferSize = that.bufferSize;
}
public int getCaching() {
@@ -885,6 +888,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.valueSize = valueSize;
}
+ public void setBufferSize(long bufferSize) {
+ this.bufferSize = bufferSize;
+ }
+
public void setPeriod(int period) {
this.period = period;
}
@@ -1020,6 +1027,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
public MemoryCompactionPolicy getInMemoryCompaction() {
return this.inMemoryCompaction;
}
+
+ public long getBufferSize() {
+ return this.bufferSize;
+ }
}
/*
@@ -1626,7 +1637,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
@Override
void onStartup() throws IOException {
- this.mutator = connection.getBufferedMutator(TableName.valueOf(opts.tableName));
+ BufferedMutatorParams p = new BufferedMutatorParams(TableName.valueOf(opts.tableName));
+ p.writeBufferSize(opts.bufferSize);
+ this.mutator = connection.getBufferedMutator(p);
this.table = connection.getTable(TableName.valueOf(opts.tableName));
}
@@ -2363,6 +2376,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
System.err.println(" asyncPrefetch Enable asyncPrefetch for scan");
System.err.println(" cacheBlocks Set the cacheBlocks option for scan. Default: true");
System.err.println(" scanReadType Set the readType option for scan, stream/pread/default. Default: default");
+ System.err.println(" bufferSize Set the value of client side buffering. Default: 2MB");
System.err.println();
System.err.println(" Note: -D properties will be applied to the conf used. ");
System.err.println(" For example: ");
@@ -2636,6 +2650,13 @@ public class PerformanceEvaluation extends Configured implements Tool {
Scan.ReadType.valueOf(cmd.substring(scanReadType.length()).toUpperCase());
continue;
}
+
+ final String bufferSize = "--bufferSize=";
+ if (cmd.startsWith(bufferSize)) {
+ opts.bufferSize = Long.parseLong(cmd.substring(bufferSize.length()));
+ continue;
+ }
+
if (isCommandClass(cmd)) {
opts.cmdName = cmd;
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/8e6ff689/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
index d5b25e2..ab93857 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
@@ -179,6 +179,16 @@ public class TestPerformanceEvaluation {
}
@Test
+ public void testSetBufferSizeOption() {
+ TestOptions opts = new PerformanceEvaluation.TestOptions();
+ long bufferSize = opts.getBufferSize();
+ assertEquals(bufferSize, 2l * 1024l * 1024l);
+ opts.setBufferSize(64l * 1024l);
+ bufferSize = opts.getBufferSize();
+ assertEquals(bufferSize, 64l * 1024l);
+ }
+
+ @Test
public void testParseOptsWithThreads() {
Queue<String> opts = new LinkedList<>();
String cmdName = "sequentialWrite";