You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/07/16 22:41:41 UTC
git commit: HBASE-2251 PE defaults to 1k rows - uncommon use case,
and easy to hit benchmarks -- Add zipf distribution of cell values
Repository: hbase
Updated Branches:
refs/heads/master 8603da0d0 -> 76543f525
HBASE-2251 PE defaults to 1k rows - uncommon use case, and easy to hit benchmarks -- Add zipf distribution of cell values
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/76543f52
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/76543f52
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/76543f52
Branch: refs/heads/master
Commit: 76543f525aa920297c5a730f35b409beddb1e90b
Parents: 8603da0
Author: stack <st...@apache.org>
Authored: Wed Jul 16 13:41:32 2014 -0700
Committer: stack <st...@apache.org>
Committed: Wed Jul 16 13:41:32 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 43 +++++++++++++++++--
.../hadoop/hbase/TestPerformanceEvaluation.java | 44 ++++++++++++++++++++
2 files changed, 84 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/76543f52/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 6db2e9a..0963ad3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -43,6 +43,7 @@ import java.util.concurrent.Future;
import com.google.common.base.Objects;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -71,6 +72,7 @@ import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.WhileMatchFilter;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.RandomDistribution;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.trace.SpanReceiverHost;
@@ -541,6 +543,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
/**
* Wraps up options passed to {@link org.apache.hadoop.hbase.PerformanceEvaluation}.
* This makes tracking all these arguments a little easier.
+ * NOTE: ADDING AN OPTION, you need to add a data member, a getter/setter (to make JSON
+ * serialization of this TestOptions class behave), and you need to add to the clone constructor
+ * below copying your new option from the 'that' to the 'this'. Look for 'clone' below.
*/
static class TestOptions {
String cmdName = null;
@@ -571,11 +576,16 @@ public class PerformanceEvaluation extends Configured implements Tool {
BloomType bloomType = BloomType.ROW;
DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
boolean valueRandom = false;
+ boolean valueZipf = false;
int valueSize = DEFAULT_VALUE_LENGTH;
int period = (this.perClientRunRows / 10) == 0? perClientRunRows: perClientRunRows / 10;
public TestOptions() {}
+ /**
+ * Clone constructor.
+ * @param that Object to copy from.
+ */
public TestOptions(TestOptions that) {
this.cmdName = that.cmdName;
this.nomapred = that.nomapred;
@@ -604,11 +614,20 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.filterAll = that.filterAll;
this.bloomType = that.bloomType;
this.valueRandom = that.valueRandom;
+ this.valueZipf = that.valueZipf;
this.valueSize = that.valueSize;
this.period = that.period;
this.randomSleep = that.randomSleep;
}
+ public boolean isValueZipf() {
+ return valueZipf;
+ }
+
+ public void setValueZipf(boolean valueZipf) {
+ this.valueZipf = valueZipf;
+ }
+
public String getCmdName() {
return cmdName;
}
@@ -877,6 +896,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
private String testName;
private Histogram latency;
private Histogram valueSize;
+ private RandomDistribution.Zipf zipf;
/**
* Note that all subclasses of this class must provide a public constructor
@@ -897,11 +917,16 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.traceSampler = Sampler.NEVER;
}
everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
+ if (options.isValueZipf()) {
+ this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.1);
+ }
LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows.");
}
int getValueLength(final Random r) {
- return opts.valueRandom? Math.abs(r.nextInt() % opts.valueSize): opts.valueSize;
+ if (this.opts.isValueRandom()) return Math.abs(r.nextInt() % opts.valueSize);
+ else if (this.opts.isValueZipf()) return Math.abs(this.zipf.nextInt());
+ else return opts.valueSize;
}
void updateValueSize(final Result [] rs) throws IOException {
@@ -1072,8 +1097,6 @@ public class PerformanceEvaluation extends Configured implements Tool {
abstract void testRow(final int i) throws IOException, InterruptedException;
}
-
- @SuppressWarnings("unused")
static class RandomSeekScanTest extends Test {
RandomSeekScanTest(HConnection con, TestOptions options, Status status) {
super(con, options, status);
@@ -1568,6 +1591,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
System.err.println(" valueSize Pass value size to use: Default: 1024");
System.err.println(" valueRandom Set if we should vary value size between 0 and " +
"'valueSize'; set on read for stats on size: Default: Not set.");
+ System.err.println(" valueZipf Set if we should vary value size between 0 and " +
+ "'valueSize' in zipf form: Default: Not set.");
System.err.println(" period Report every 'period' rows: " +
"Default: opts.perClientRunRows / 10");
System.err.println(" multiGet Batch gets together into groups of N. Only supported " +
@@ -1766,6 +1791,18 @@ public class PerformanceEvaluation extends Configured implements Tool {
final String valueRandom = "--valueRandom";
if (cmd.startsWith(valueRandom)) {
opts.valueRandom = true;
+ if (opts.valueZipf) {
+ throw new IllegalStateException("Either valueZipf or valueRandom but not both");
+ }
+ continue;
+ }
+
+ final String valueZipf = "--valueZipf";
+ if (cmd.startsWith(valueZipf)) {
+ opts.valueZipf = true;
+ if (opts.valueRandom) {
+ throw new IllegalStateException("Either valueZipf or valueRandom but not both");
+ }
continue;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/76543f52/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
new file mode 100644
index 0000000..a5bfcbe
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestPerformanceEvaluation {
+ @Test
+ public void testSerialization()
+ throws JsonGenerationException, JsonMappingException, IOException {
+ PerformanceEvaluation.TestOptions options = new PerformanceEvaluation.TestOptions();
+ assertTrue(!options.isAutoFlush());
+ options.setAutoFlush(true);
+ ObjectMapper mapper = new ObjectMapper();
+ String optionsString = mapper.writeValueAsString(options);
+ PerformanceEvaluation.TestOptions optionsDeserialized =
+ mapper.readValue(optionsString, PerformanceEvaluation.TestOptions.class);
+ assertTrue(optionsDeserialized.isAutoFlush());
+ }
+}
\ No newline at end of file