You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nk...@apache.org on 2014/04/16 22:51:05 UTC
svn commit: r1588069 -
/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
Author: nkeywal
Date: Wed Apr 16 20:51:05 2014
New Revision: 1588069
URL: http://svn.apache.org/r1588069
Log:
HBASE-11000 Add autoflush option to PerformanceEvaluation
Modified:
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java?rev=1588069&r1=1588068&r2=1588069&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java Wed Apr 16 20:51:05 2014
@@ -43,7 +43,6 @@ import com.yammer.metrics.core.MetricsRe
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -117,7 +116,6 @@ public class PerformanceEvaluation exten
private static final MathContext CXT = MathContext.DECIMAL64;
private static final BigDecimal MS_PER_SEC = BigDecimal.valueOf(1000);
private static final BigDecimal BYTES_PER_MB = BigDecimal.valueOf(1024 * 1024);
- private static final TestOptions DEFAULT_OPTS = new TestOptions();
protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>();
@@ -237,7 +235,7 @@ public class PerformanceEvaluation exten
Configuration conf = HBaseConfiguration.create(context.getConfiguration());
// Evaluation task
- long elapsedTime = this.pe.runOneClient(this.cmd, conf, opts, status);
+ long elapsedTime = runOneClient(this.cmd, conf, opts, status);
// Collect how much time the thing took. Report as map output and
// to the ELAPSED_TIME counter.
context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime);
@@ -353,8 +351,8 @@ public class PerformanceEvaluation exten
+ Arrays.toString(timings));
Arrays.sort(timings);
long total = 0;
- for (int i = 0; i < timings.length; i++) {
- total += timings[i];
+ for (long timing : timings) {
+ total += timing;
}
LOG.info("[" + test + "]"
+ "\tMin: " + timings[0] + "ms"
@@ -493,6 +491,7 @@ public class PerformanceEvaluation exten
this.tableName = that.tableName;
this.flushCommits = that.flushCommits;
this.writeToWAL = that.writeToWAL;
+ this.autoFlush = that.autoFlush;
this.useTags = that.useTags;
this.noOfTags = that.noOfTags;
this.reportLatency = that.reportLatency;
@@ -513,6 +512,7 @@ public class PerformanceEvaluation exten
public String tableName = TABLE_NAME;
public boolean flushCommits = true;
public boolean writeToWAL = true;
+ public boolean autoFlush = false;
public boolean useTags = false;
public int noOfTags = 1;
public boolean reportLatency = false;
@@ -574,7 +574,7 @@ public class PerformanceEvaluation exten
void testSetup() throws IOException {
this.connection = HConnectionManager.createConnection(conf);
this.table = connection.getTable(opts.tableName);
- this.table.setAutoFlush(false, true);
+ this.table.setAutoFlush(opts.autoFlush, true);
String metricName =
testName + "-Client-" + Thread.currentThread().getName() + "-testRowTime";
latency =
@@ -615,9 +615,9 @@ public class PerformanceEvaluation exten
// Report on completion of 1/10th of total.
for (int i = opts.startRow; i < lastRow; i++) {
if (i % everyN != 0) continue;
- long startTime = System.currentTimeMillis();
+ long startTime = System.nanoTime();
testRow(i);
- latency.update(System.currentTimeMillis() - startTime);
+ latency.update((System.nanoTime() - startTime) / 100000);
if (status != null && i > 0 && (i % getReportingPeriod()) == 0) {
status.setStatus(generateStatus(opts.startRow, i, lastRow));
}
@@ -629,14 +629,16 @@ public class PerformanceEvaluation exten
*/
private void reportLatency() throws IOException {
status.setStatus(testName + " latency log (ms), on " + latency.count() + " measures");
- status.setStatus(testName + " Min = " + latency.min());
- status.setStatus(testName + " Avg = " + latency.mean());
- status.setStatus(testName + " StdDev = " + latency.stdDev());
- status.setStatus(testName + " 50th = " + latency.getSnapshot().getMedian());
- status.setStatus(testName + " 95th = " + latency.getSnapshot().get95thPercentile());
- status.setStatus(testName + " 99th = " + latency.getSnapshot().get99thPercentile());
- status.setStatus(testName + " 99.9th = " + latency.getSnapshot().get999thPercentile());
- status.setStatus(testName + " Max = " + latency.max());
+ status.setStatus(testName + " Min = " + latency.min());
+ status.setStatus(testName + " Avg = " + latency.mean());
+ status.setStatus(testName + " StdDev = " + latency.stdDev());
+ status.setStatus(testName + " 50th = " + latency.getSnapshot().getMedian());
+ status.setStatus(testName + " 95th = " + latency.getSnapshot().get95thPercentile());
+ status.setStatus(testName + " 99th = " + latency.getSnapshot().get99thPercentile());
+ status.setStatus(testName + " 99.9th = " + latency.getSnapshot().get999thPercentile());
+ status.setStatus(testName + " 99.99th = " + latency.getSnapshot().getValue(0.9999));
+ status.setStatus(testName + " 99.9999th= " + latency.getSnapshot().getValue(0.99999));
+ status.setStatus(testName + " Max = " + latency.max());
}
/*
@@ -684,7 +686,7 @@ public class PerformanceEvaluation exten
scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
ResultScanner s = this.table.getScanner(scan);
int count = 0;
- for (Result rr; (rr = s.next()) != null;) {
+ while (s.next() != null) {
count++;
}
@@ -758,7 +760,6 @@ public class PerformanceEvaluation exten
static class RandomReadTest extends Test {
private ArrayList<Get> gets;
- int idx = 0;
RandomReadTest(Configuration conf, TestOptions options, Status status) {
super(conf, options, status);
@@ -973,7 +974,7 @@ public class PerformanceEvaluation exten
*/
public static byte[] generateData(final Random r, int length) {
byte [] b = new byte [length];
- int i = 0;
+ int i;
for(i = 0; i < (length-8); i += 8) {
b[i] = (byte) (65 + r.nextInt(26));
@@ -1002,7 +1003,7 @@ public class PerformanceEvaluation exten
throws IOException {
status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " +
opts.perClientRunRows + " rows");
- long totalElapsedTime = 0;
+ long totalElapsedTime;
final Test t;
try {
@@ -1066,6 +1067,7 @@ public class PerformanceEvaluation exten
System.err.println(" flushCommits Used to determine if the test should flush the table. " +
"Default: false");
System.err.println(" writeToWAL Set writeToWAL on puts. Default: True");
+ System.err.println(" autoFlush Set autoFlush on htable. Default: False");
System.err.println(" presplit Create presplit table. Recommended for accurate perf " +
"analysis (see guide). Default: disabled");
System.err.println(" inmemory Tries to keep the HFiles of the CF " +
@@ -1075,8 +1077,7 @@ public class PerformanceEvaluation exten
"Default: false");
System.err.println(" numoftags Specify the no of tags that would be needed. " +
"This works only if usetags is true.");
- System.err.println(" latency Set to report operation latencies. " +
- "Currently only supported by randomRead test. Default: False");
+ System.err.println(" latency Set to report operation latencies. Default: False");
System.err.println();
System.err.println(" Note: -D properties will be applied to the conf used. ");
System.err.println(" For example: ");
@@ -1183,6 +1184,12 @@ public class PerformanceEvaluation exten
continue;
}
+ final String autoFlush = "--autoFlush=";
+ if (cmd.startsWith(autoFlush)) {
+ opts.autoFlush = Boolean.parseBoolean(cmd.substring(autoFlush.length()));
+ continue;
+ }
+
final String presplit = "--presplit=";
if (cmd.startsWith(presplit)) {
opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length()));