You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2014/02/19 18:22:12 UTC
svn commit: r1569830 -
/hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
Author: ndimiduk
Date: Wed Feb 19 17:22:12 2014
New Revision: 1569830
URL: http://svn.apache.org/r1569830
Log:
HBASE-10419 Add multiget support to PerformanceEvaluation
Modified:
hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
Modified: hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java?rev=1569830&r1=1569829&r2=1569830&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java (original)
+++ hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java Wed Feb 19 17:22:12 2014
@@ -133,6 +133,7 @@ public class PerformanceEvaluation exten
private boolean inMemoryCF = false;
private boolean reportLatency = false;
private int presplitRegions = 0;
+ private int multiGet = 0;
private HConnection connection;
private static final Path PERF_EVAL_DIR = new Path("performance_evaluation");
@@ -147,7 +148,8 @@ public class PerformanceEvaluation exten
"clients=(\\d+),\\s+" +
"flushCommits=(\\w+),\\s+" +
"writeToWAL=(\\w+),\\s+" +
- "reportLatency=(\\w+)");
+ "reportLatency=(\\w+),\\s+" +
+ "multiGet=(\\d+)");
/**
* Enum for map metrics. Keep it out here rather than inside in the Map
@@ -226,12 +228,13 @@ public class PerformanceEvaluation exten
private boolean flushCommits = false;
private boolean writeToWAL = true;
private boolean reportLatency = false;
+ private int multiGet = 0;
public PeInputSplit() {}
public PeInputSplit(TableName tableName, int startRow, int rows, int totalRows,
float sampleRate, int clients, boolean flushCommits, boolean writeToWAL,
- boolean reportLatency) {
+ boolean reportLatency, int multiGet) {
this.tableName = tableName;
this.startRow = startRow;
this.rows = rows;
@@ -241,6 +244,7 @@ public class PerformanceEvaluation exten
this.flushCommits = flushCommits;
this.writeToWAL = writeToWAL;
this.reportLatency = reportLatency;
+ this.multiGet = multiGet;
}
@Override
@@ -258,6 +262,7 @@ public class PerformanceEvaluation exten
this.flushCommits = in.readBoolean();
this.writeToWAL = in.readBoolean();
this.reportLatency = in.readBoolean();
+ this.multiGet = in.readInt();
}
@Override
@@ -273,6 +278,7 @@ public class PerformanceEvaluation exten
out.writeBoolean(flushCommits);
out.writeBoolean(writeToWAL);
out.writeBoolean(reportLatency);
+ out.writeInt(multiGet);
}
@Override
@@ -320,6 +326,10 @@ public class PerformanceEvaluation exten
public boolean isReportLatency() {
return reportLatency;
}
+
+ public int getMultiGet() {
+ return multiGet;
+ }
}
/**
@@ -359,6 +369,7 @@ public class PerformanceEvaluation exten
boolean flushCommits = Boolean.parseBoolean(m.group(7));
boolean writeToWAL = Boolean.parseBoolean(m.group(8));
boolean reportLatency = Boolean.parseBoolean(m.group(9));
+ int multiGet = Integer.parseInt(m.group(10));
LOG.debug("tableName=" + tableName +
" split["+ splitList.size() + "] " +
@@ -369,11 +380,12 @@ public class PerformanceEvaluation exten
" clients=" + clients +
" flushCommits=" + flushCommits +
" writeToWAL=" + writeToWAL +
- " reportLatency=" + reportLatency);
+ " reportLatency=" + reportLatency +
+ " multiGet=" + multiGet);
PeInputSplit newSplit =
new PeInputSplit(tableName, startRow, rows, totalRows, sampleRate, clients,
- flushCommits, writeToWAL, reportLatency);
+ flushCommits, writeToWAL, reportLatency, multiGet);
splitList.add(newSplit);
}
}
@@ -496,7 +508,8 @@ public class PerformanceEvaluation exten
long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(),
value.getRows(), value.getTotalRows(), value.getSampleRate(),
value.isFlushCommits(), value.isWriteToWAL(), value.isReportLatency(),
- HConnectionManager.createConnection(context.getConfiguration()), status);
+ value.getMultiGet(), HConnectionManager.createConnection(context.getConfiguration()),
+ status);
// Collect how much time the thing took. Report as map output and
// to the ELAPSED_TIME counter.
context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime);
@@ -604,6 +617,7 @@ public class PerformanceEvaluation exten
final boolean writeToWal = this.writeToWAL;
final boolean reportLatency = this.reportLatency;
final int preSplitRegions = this.presplitRegions;
+ final int multiGet = this.multiGet;
final HConnection connection = HConnectionManager.createConnection(getConf());
for (int i = 0; i < this.N; i++) {
final int index = i;
@@ -622,10 +636,11 @@ public class PerformanceEvaluation exten
pe.sampleRate = sampleRate;
pe.reportLatency = reportLatency;
pe.connection = connection;
+ pe.multiGet = multiGet;
try {
long elapsedTime = pe.runOneClient(cmd, index * perClientRows,
- perClientRows, R, sampleRate,
- flushCommits, writeToWAL, reportLatency, connection, new Status() {
+ perClientRows, R, sampleRate, flushCommits, writeToWAL, reportLatency, multiGet,
+ connection, new Status() {
public void setStatus(final String msg) throws IOException {
LOG.info("client-" + getName() + " " + msg);
}
@@ -738,7 +753,8 @@ public class PerformanceEvaluation exten
", clients=" + this.N +
", flushCommits=" + this.flushCommits +
", writeToWAL=" + this.writeToWAL +
- ", reportLatency=" + this.reportLatency;
+ ", reportLatency=" + this.reportLatency +
+ ", multiGet=" + this.multiGet;
int hash = h.hash(Bytes.toBytes(s));
m.put(hash, s);
}
@@ -793,11 +809,12 @@ public class PerformanceEvaluation exten
private boolean flushCommits;
private boolean writeToWAL = true;
private boolean reportLatency;
+ private int multiGet = 0;
private HConnection connection;
TestOptions(int startRow, int perClientRunRows, int totalRows, float sampleRate,
int numClientThreads, TableName tableName, boolean flushCommits, boolean writeToWAL,
- boolean reportLatency, HConnection connection) {
+ boolean reportLatency, int multiGet, HConnection connection) {
this.startRow = startRow;
this.perClientRunRows = perClientRunRows;
this.totalRows = totalRows;
@@ -807,6 +824,7 @@ public class PerformanceEvaluation exten
this.flushCommits = flushCommits;
this.writeToWAL = writeToWAL;
this.reportLatency = reportLatency;
+ this.multiGet = multiGet;
this.connection = connection;
}
@@ -846,6 +864,10 @@ public class PerformanceEvaluation exten
return reportLatency;
}
+ public int getMultiGet() {
+ return multiGet;
+ }
+
public HConnection getConnection() {
return connection;
}
@@ -1071,6 +1093,8 @@ public class PerformanceEvaluation exten
private final int everyN;
private final boolean reportLatency;
private final double[] times;
+ private final int multiGet;
+ private ArrayList<Get> gets;
int idx = 0;
RandomReadTest(Configuration conf, TestOptions options, Status status) {
@@ -1078,10 +1102,15 @@ public class PerformanceEvaluation exten
everyN = (int) (this.totalRows / (this.totalRows * this.sampleRate));
LOG.info("Sampling 1 every " + everyN + " out of " + perClientRunRows + " total rows.");
this.reportLatency = options.isReportLatency();
+ this.multiGet = options.getMultiGet();
+ if (this.multiGet > 0) {
+ LOG.info("MultiGet enabled. Sending GETs in batches of " + this.multiGet + ".");
+ this.gets = new ArrayList<Get>(this.multiGet);
+ }
if (this.reportLatency) {
- times = new double[(int) Math.ceil(this.perClientRunRows * this.sampleRate)];
+ this.times = new double[(int) Math.ceil(this.perClientRunRows * this.sampleRate / Math.max(1, this.multiGet))];
} else {
- times = null;
+ this.times = null;
}
}
@@ -1090,10 +1119,22 @@ public class PerformanceEvaluation exten
if (i % everyN == 0) {
Get get = new Get(getRandomRow(this.rand, this.totalRows));
get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
- long start = System.nanoTime();
- this.table.get(get);
- if (this.reportLatency) {
- times[idx++] = (float) ((System.nanoTime() - start) / 1000000.0);
+ if (this.multiGet > 0) {
+ this.gets.add(get);
+ if (this.gets.size() == this.multiGet) {
+ long start = System.nanoTime();
+ this.table.get(this.gets);
+ if (this.reportLatency) {
+ times[idx++] = (System.nanoTime() - start) / 1e6;
+ }
+ this.gets.clear();
+ }
+ } else {
+ long start = System.nanoTime();
+ this.table.get(get);
+ if (this.reportLatency) {
+ times[idx++] = (System.nanoTime() - start) / 1e6;
+ }
}
}
}
@@ -1106,6 +1147,10 @@ public class PerformanceEvaluation exten
@Override
protected void testTakedown() throws IOException {
+ if (this.gets != null && this.gets.size() > 0) {
+ this.table.get(gets);
+ this.gets.clear();
+ }
super.testTakedown();
if (this.reportLatency) {
Arrays.sort(times);
@@ -1302,7 +1347,7 @@ public class PerformanceEvaluation exten
long runOneClient(final Class<? extends Test> cmd, final int startRow,
final int perClientRunRows, final int totalRows, final float sampleRate,
- boolean flushCommits, boolean writeToWAL, boolean reportLatency,
+ boolean flushCommits, boolean writeToWAL, boolean reportLatency, int multiGet,
HConnection connection, final Status status)
throws IOException {
status.setStatus("Start " + cmd + " at offset " + startRow + " for " +
@@ -1311,7 +1356,7 @@ public class PerformanceEvaluation exten
TestOptions options = new TestOptions(startRow, perClientRunRows,
totalRows, sampleRate, N, tableName, flushCommits, writeToWAL,
- reportLatency, connection);
+ reportLatency, multiGet, connection);
final Test t;
try {
Constructor<? extends Test> constructor = cmd.getDeclaredConstructor(
@@ -1345,7 +1390,7 @@ public class PerformanceEvaluation exten
admin = new HBaseAdmin(getConf());
checkTable(admin);
runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits,
- this.writeToWAL, this.reportLatency, this.connection, status);
+ this.writeToWAL, this.reportLatency, this.multiGet, this.connection, status);
} catch (Exception e) {
LOG.error("Failed", e);
} finally {
@@ -1519,6 +1564,12 @@ public class PerformanceEvaluation exten
continue;
}
+ final String multiGet = "--multiGet=";
+ if (cmd.startsWith(multiGet)) {
+ this.multiGet = Integer.parseInt(cmd.substring(multiGet.length()));
+ continue;
+ }
+
this.connection = HConnectionManager.createConnection(getConf());
Class<? extends Test> cmdClass = determineCommandClass(cmd);