You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by uj...@apache.org on 2014/05/24 01:14:33 UTC
[2/2] git commit: ACCUMULO-2789 Added parameter to control the number
of cells within a row
ACCUMULO-2789 Added parameter to control the number of cells within a row
When running the stress test, it is now possible to configure the writer to
send mutations for rows that contain a random amount of column updates. This is
controlled via the --min-row-width, --max-row-width and --row-width-seed
parameters.
I did some internal shuffling to make randomly generating numbers within a
range more convenient. See the new RandomWithinRange class for details.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f0dcf1e5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f0dcf1e5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f0dcf1e5
Branch: refs/heads/1.5.2-SNAPSHOT
Commit: f0dcf1e5570c12098e716ec58ea2f4bc3321053b
Parents: 6970d73
Author: Bill Slacum <uj...@apache.org>
Authored: Fri May 23 19:11:39 2014 -0400
Committer: Bill Slacum <uj...@apache.org>
Committed: Fri May 23 19:11:39 2014 -0400
----------------------------------------------------------------------
.../random/RandomByteArrayMakerFunction.java | 60 --------------------
.../test/stress/random/RandomByteArrays.java | 14 ++---
.../test/stress/random/RandomMutations.java | 11 +++-
.../test/stress/random/RandomWithinRange.java | 43 ++++++++++++++
.../accumulo/test/stress/random/Write.java | 33 +++++++----
.../test/stress/random/WriteOptions.java | 17 ++++++
test/system/stress/writer.sh | 4 +-
7 files changed, 97 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java
deleted file mode 100644
index bfe0be8..0000000
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrayMakerFunction.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.stress.random;
-
-import java.util.Random;
-
-import com.google.common.base.Preconditions;
-
-/**
- * This class is one RBAMF. Don't mess with it, or it'll mess with you.
- */
-public class RandomByteArrayMakerFunction {
- private final Random random;
-
- public RandomByteArrayMakerFunction() {
- this(0);
- }
-
- public RandomByteArrayMakerFunction(int seed) {
- random = new Random(seed);
- }
-
- public byte[] make(int size) {
- byte[] b = new byte[size];
- random.nextBytes(b);
- return b;
- }
-
- public byte[] makeWithRandomSize(int max) {
- return makeWithRandomSize(1, random.nextInt(max));
- }
-
- public byte[] makeWithRandomSize(int min, int max) {
- Preconditions.checkArgument(min > 0, "Min must be positive.");
- Preconditions.checkArgument(max >= min, "Max must be greater than or equal to min.");
- if (min == max) {
- return make(min);
- } else {
- final int spread = max - min;
- final int random_value = random.nextInt(spread);
- // we pick a random number that's between 0 and (max - min), then add
- // min as an offset to get a random number that's [min, max)
- return make(random_value + min);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java
index 62f7cb2..0b6b36a 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomByteArrays.java
@@ -17,20 +17,16 @@
package org.apache.accumulo.test.stress.random;
/**
- * An iterator that will create random byte arrays as it is looped over.
+ * A stream that will create random byte arrays as it is looped over.
*/
public class RandomByteArrays extends Stream<byte[]> {
- private final RandomByteArrayMakerFunction rbamf;
- private final int min_array_size, max_array_size;
+ private final RandomWithinRange random_arrays;
- public RandomByteArrays(RandomByteArrayMakerFunction rbamf, int min_array_size,
- int max_array_size) {
- this.rbamf = rbamf;
- this.min_array_size = min_array_size;
- this.max_array_size = max_array_size;
+ public RandomByteArrays(RandomWithinRange random_arrays) {
+ this.random_arrays = random_arrays;
}
public byte[] next() {
- return rbamf.makeWithRandomSize(min_array_size, max_array_size);
+ return random_arrays.next_bytes();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
index a22222b..c4504b2 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomMutations.java
@@ -20,20 +20,25 @@ import org.apache.accumulo.core.data.Mutation;
public class RandomMutations extends Stream<Mutation> {
private final RandomByteArrays rows, column_families, column_qualifiers, values;
+ private final RandomWithinRange row_widths;
public RandomMutations(RandomByteArrays rows, RandomByteArrays column_families,
- RandomByteArrays column_qualifiers, RandomByteArrays values) {
+ RandomByteArrays column_qualifiers, RandomByteArrays values, RandomWithinRange row_widths) {
this.rows = rows;
this.column_families = column_families;
this.column_qualifiers = column_qualifiers;
this.values = values;
+ this.row_widths = row_widths;
}
-
+
// TODO should we care about timestamps?
@Override
public Mutation next() {
Mutation m = new Mutation(rows.next());
- m.put(column_families.next(), column_qualifiers.next(), values.next());
+ final int cells = row_widths.next();
+ for(int i = 0; i < cells; ++i) {
+ m.put(column_families.next(), column_qualifiers.next(), values.next());
+ }
return m;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java
new file mode 100644
index 0000000..c66cf38
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/RandomWithinRange.java
@@ -0,0 +1,43 @@
+package org.apache.accumulo.test.stress.random;
+
+import java.util.Random;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Class that returns positive integers between some minimum
+ * and maximum.
+ *
+ */
+public class RandomWithinRange {
+ private final Random random;
+ private final int min, max;
+
+ public RandomWithinRange(int seed, int min, int max) {
+ this(new Random(seed), min, max);
+ }
+
+ public RandomWithinRange(Random random, int min, int max) {
+ Preconditions.checkArgument(min > 0, "Min must be positive.");
+ Preconditions.checkArgument(max >= min, "Max must be greater than or equal to min.");
+ this.random = random;
+ this.min = min;
+ this.max = max;
+ }
+
+ public int next() {
+ if (min == max) {
+ return min;
+ } else {
+ // we pick a random number that's between 0 and (max - min), then add
+ // min as an offset to get a random number that's [min, max)
+ return random.nextInt(max - min) + min;
+ }
+ }
+
+ public byte[] next_bytes() {
+ byte[] b = new byte[next()];
+ random.nextBytes(b);
+ return b;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
index 2c817c3..3df9808 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
@@ -52,24 +52,33 @@ public class Write {
new RandomMutations(
//rows
new RandomByteArrays(
- new RandomByteArrayMakerFunction(opts.row_seed),
- opts.rowMin(),
- opts.rowMax()),
+ new RandomWithinRange(
+ opts.row_seed,
+ opts.rowMin(),
+ opts.rowMax())),
//cfs
new RandomByteArrays(
- new RandomByteArrayMakerFunction(opts.cf_seed),
- opts.cfMin(),
- opts.cfMax()),
+ new RandomWithinRange(
+ opts.cf_seed,
+ opts.cfMin(),
+ opts.cfMax())),
//cqs
new RandomByteArrays(
- new RandomByteArrayMakerFunction(opts.cq_seed),
- opts.cqMin(),
- opts.cqMax()),
+ new RandomWithinRange(
+ opts.cq_seed,
+ opts.cqMin(),
+ opts.cqMax())),
//vals
new RandomByteArrays(
- new RandomByteArrayMakerFunction(opts.value_seed),
- opts.valueMin(),
- opts.valueMax())));
+ new RandomWithinRange(
+ opts.value_seed,
+ opts.valueMin(),
+ opts.valueMax())),
+ //number of cells per row
+ new RandomWithinRange(
+ opts.row_width_seed,
+ opts.rowWidthMin(),
+ opts.rowWidthMax())));
while(true) {
dw.next();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
index b850e85..85ff25b 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
@@ -48,6 +48,12 @@ class WriteOptions extends ClientOnDefaultTable {
@Parameter(validateValueWith = IntArgValidator.class, names = "--max-value-size", description = "maximum value size")
Integer value_max;
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--min-row-width", description = "minimum row width")
+ Integer row_width_min;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--max-row-width", description = "maximum row width")
+ Integer row_width_max;
+
@Parameter(names = "--clear-table", description = "clears the table before ingesting")
boolean clear_table;
@@ -63,6 +69,9 @@ class WriteOptions extends ClientOnDefaultTable {
@Parameter(names = "--value-seed", description = "seed for generating values")
int value_seed = 99;
+ @Parameter(names = "--row-width-seed", description = "seed for generating the number of cells within a row (a row's \"width\")")
+ int row_width_seed = 444;
+
public WriteOptions(String table) {
super(table);
}
@@ -149,4 +158,12 @@ class WriteOptions extends ClientOnDefaultTable {
public int valueMax() {
return calculateMax(value_min, value_max);
}
+
+ public int rowWidthMin() {
+ return minOrDefault(row_width_min);
+ }
+
+ public int rowWidthMax() {
+ return calculateMax(row_width_min, row_width_max);
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f0dcf1e5/test/system/stress/writer.sh
----------------------------------------------------------------------
diff --git a/test/system/stress/writer.sh b/test/system/stress/writer.sh
index 6054ab1..8265a68 100755
--- a/test/system/stress/writer.sh
+++ b/test/system/stress/writer.sh
@@ -24,15 +24,17 @@ ROW_RANGE='--min-row-size 128 --max-row-size 128'
CF_RANGE='--min-cf-size 128 --max-cf-size 128'
CQ_RANGE='--min-cq-size 128 --max-cq-size 128'
VALUE_RANGE='--min-value-size 1024 --max-value-size 2048'
+ROW_WIDTH='--min-row-width 1 --max-row-width 10'
# These are the seeds for the random number generates used to generate each cell component.
ROW_SEED='--row-seed 1'
CF_SEED='--cf-seed 2'
CQ_SEED='--cq-seed 3'
VALUE_SEED='--value-seed 4'
+ROW_WIDTH_SEED='--row-width-seed 5'
# Let's reset the table, for good measure
../../../bin/accumulo shell $USERPASS -e 'deletetable -f stress_test'
../../../bin/accumulo shell $USERPASS -e 'createtable stress_test'
-../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED
+../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED