You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bh...@apache.org on 2014/06/26 17:09:56 UTC
[2/6] git commit: ACCUMULO-2849 Add scanner batch size and write
delay configurability to mem stress test
ACCUMULO-2849 Add scanner batch size and write delay configurability to mem stress test
This commit adds one new option each to the Scan and Write components of the memory
stress test. The --scan-batch-size option for Scan configures the batch size of the
scanner it uses, while the --write-delay option for Write allows for a delay between
each mutation added to its batch writer. The reader.sh and writer.sh scripts are
updated for easy configuration of these new options.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f7fe2a84
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f7fe2a84
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f7fe2a84
Branch: refs/heads/1.6.1-SNAPSHOT
Commit: f7fe2a8475bf53da6f6f6e8374a39af0d4d9acbb
Parents: 886cd19
Author: Bill Havanki <bh...@cloudera.com>
Authored: Tue Jun 24 21:17:57 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Tue Jun 24 21:17:57 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/test/stress/random/Scan.java | 3 +++
.../org/apache/accumulo/test/stress/random/ScanOpts.java | 5 ++++-
.../org/apache/accumulo/test/stress/random/Write.java | 10 +++++++++-
.../apache/accumulo/test/stress/random/WriteOptions.java | 5 ++++-
test/system/stress/reader.sh | 6 +++++-
test/system/stress/writer.sh | 6 +++++-
6 files changed, 30 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java b/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java
index 00a1cec..9408770 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/Scan.java
@@ -54,6 +54,9 @@ public class Scan {
Range range = pickRange(connector.tableOperations(), opts.getTableName(),
tablet_index_generator);
scanner.setRange(range);
+ if (opts.batch_size > 0) {
+ scanner.setBatchSize(opts.batch_size);
+ }
try {
consume(scanner);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java b/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java
index 22024d1..86e7920 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/ScanOpts.java
@@ -33,7 +33,10 @@ class ScanOpts extends ClientOnDefaultTable {
@Parameter(names = "--scan-seed", description = "seed for randomly choosing tablets to scan")
int scan_seed = 1337;
-
+
+ @Parameter(names = "--scan-batch-size", description="scanner batch size")
+ int batch_size = -1;
+
public ScanOpts() {
this(WriteOptions.DEFAULT_TABLE);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
index 3df9808..9c29871 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/Write.java
@@ -47,7 +47,12 @@ public class Write {
System.err.println("Couldn't create table ourselves, but that's ok. Continuing.");
}
}
-
+
+ long writeDelay = opts.write_delay;
+ if (writeDelay < 0) {
+ writeDelay = 0;
+ }
+
DataWriter dw = new DataWriter(c.createBatchWriter(opts.getTableName(), batch_writer_opts.getBatchWriterConfig()),
new RandomMutations(
//rows
@@ -82,6 +87,9 @@ public class Write {
while(true) {
dw.next();
+ if (writeDelay > 0) {
+ Thread.sleep(writeDelay);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
index 85ff25b..c213528 100644
--- a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
@@ -71,7 +71,10 @@ class WriteOptions extends ClientOnDefaultTable {
@Parameter(names = "--row-width-seed", description = "seed for generating the number of cells within a row (a row's \"width\")")
int row_width_seed = 444;
-
+
+ @Parameter(names = "--write-delay", description = "milliseconds to wait between writes")
+ long write_delay = 0L;
+
public WriteOptions(String table) {
super(table);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/system/stress/reader.sh
----------------------------------------------------------------------
diff --git a/test/system/stress/reader.sh b/test/system/stress/reader.sh
index 8f10fa2..0b796f0 100755
--- a/test/system/stress/reader.sh
+++ b/test/system/stress/reader.sh
@@ -31,4 +31,8 @@ CONTINUOUS_SCAN='--continuous'
# Controls whether or not the scan will be using an isolated scanner. Add this to the execution
#SCAN_ISOLATION='--isolate'
-../../../bin/accumulo org.apache.accumulo.test.stress.random.Scan $INSTANCE $USERPASS $SCAN_SEED $CONTINUOUS_SCAN
\ No newline at end of file
+# Sets the batch size for the scanner, use a lower number for large rows / cells
+#SCAN_BATCH_SIZE='--scan-batch-size -1'
+
+../../../bin/accumulo org.apache.accumulo.test.stress.random.Scan $INSTANCE $USERPASS $SCAN_SEED $CONTINUOUS_SCAN \
+ $SCAN_BATCH_SIZE
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7fe2a84/test/system/stress/writer.sh
----------------------------------------------------------------------
diff --git a/test/system/stress/writer.sh b/test/system/stress/writer.sh
index 8265a68..db55155 100755
--- a/test/system/stress/writer.sh
+++ b/test/system/stress/writer.sh
@@ -33,8 +33,12 @@ CQ_SEED='--cq-seed 3'
VALUE_SEED='--value-seed 4'
ROW_WIDTH_SEED='--row-width-seed 5'
+# This is the delay in milliseconds between writes. Use <= 0 for no delay.
+WRITE_DELAY='--write-delay 0'
+
# Let's reset the table, for good measure
../../../bin/accumulo shell $USERPASS -e 'deletetable -f stress_test'
../../../bin/accumulo shell $USERPASS -e 'createtable stress_test'
-../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED
+../../../bin/accumulo org.apache.accumulo.test.stress.random.Write $INSTANCE $USERPASS $ROW_RANGE $CF_RANGE $CQ_RANGE $VALUE_RANGE \
+ $ROW_SEED $CF_SEED $CQ_SEED $VALUE_SEED $ROW_WIDTH $ROW_WIDTH_SEED $WRITE_DELAY