You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2012/06/15 22:19:32 UTC
svn commit: r1350779 - in /accumulo/trunk:
server/src/main/java/org/apache/accumulo/server/test/continuous/
test/system/continuous/
Author: kturner
Date: Fri Jun 15 20:19:31 2012
New Revision: 1350779
URL: http://svn.apache.org/viewvc?rev=1350779&view=rev
Log:
ACCUMULO-637 a few continuous ingest changes: made # entries written configurable, made table creation automatic, made log dir creation automatic
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
accumulo/trunk/test/system/continuous/continuous-env.sh.example
accumulo/trunk/test/system/continuous/start-batchwalkers.sh
accumulo/trunk/test/system/continuous/start-ingest.sh
accumulo/trunk/test/system/continuous/start-scanners.sh
accumulo/trunk/test/system/continuous/start-stats.sh
accumulo/trunk/test/system/continuous/start-walkers.sh
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousIngest.java Fri Jun 15 20:19:31 2012
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.B
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -67,11 +68,11 @@ public class ContinuousIngest {
args = processOptions(args);
- if (args.length != 13) {
+ if (args.length != 14) {
throw new IllegalArgumentException(
"usage : "
+ ContinuousIngest.class.getName()
- + " [--debug <debug log>] <instance name> <zookeepers> <user> <pass> <table> <min> <max> <max colf> <max colq> <max mem> <max latency> <max threads> <enable checksum>");
+ + " [--debug <debug log>] <instance name> <zookeepers> <user> <pass> <table> <num> <min> <max> <max colf> <max colq> <max mem> <max latency> <max threads> <enable checksum>");
}
if (debugLog != null) {
@@ -89,16 +90,17 @@ public class ContinuousIngest {
String table = args[4];
- long min = Long.parseLong(args[5]);
- long max = Long.parseLong(args[6]);
- short maxColF = Short.parseShort(args[7]);
- short maxColQ = Short.parseShort(args[8]);
-
- long maxMemory = Long.parseLong(args[9]);
- long maxLatency = Integer.parseInt(args[10]);
- int maxWriteThreads = Integer.parseInt(args[11]);
+ long num = Long.parseLong(args[5]);
+ long min = Long.parseLong(args[6]);
+ long max = Long.parseLong(args[7]);
+ short maxColF = Short.parseShort(args[8]);
+ short maxColQ = Short.parseShort(args[9]);
+
+ long maxMemory = Long.parseLong(args[10]);
+ long maxLatency = Integer.parseInt(args[11]);
+ int maxWriteThreads = Integer.parseInt(args[12]);
- boolean checksum = Boolean.parseBoolean(args[12]);
+ boolean checksum = Boolean.parseBoolean(args[13]);
if (min < 0 || max < 0 || max <= min) {
throw new IllegalArgumentException("bad min and max");
@@ -109,6 +111,11 @@ public class ContinuousIngest {
String path = ZooUtil.getRoot(instance) + Constants.ZTRACERS;
Tracer.getInstance().addReceiver(new ZooSpanClient(zooKeepers, path, localhost, "cingest", 1000));
+ if (!conn.tableOperations().exists(table))
+ try {
+ conn.tableOperations().create(table);
+ } catch (TableExistsException tee) {}
+
BatchWriter bw = conn.createBatchWriter(table, maxMemory, maxLatency, maxWriteThreads);
bw = Trace.wrapAll(bw, new CountSampler(1024));
@@ -133,7 +140,7 @@ public class ContinuousIngest {
long lastFlushTime = System.currentTimeMillis();
- while (true) {
+ out: while (true) {
// generate first set of nodes
for (int index = 0; index < flushInterval; index++) {
long rowLong = genLong(min, max, r);
@@ -152,6 +159,8 @@ public class ContinuousIngest {
}
lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
+ if (count >= num)
+ break out;
// generate subsequent sets of nodes that link to previous set of nodes
for (int depth = 1; depth < maxDepth; depth++) {
@@ -165,6 +174,8 @@ public class ContinuousIngest {
}
lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
+ if (count >= num)
+ break out;
}
// create one big linked list, this makes all of the first inserts
@@ -175,6 +186,8 @@ public class ContinuousIngest {
bw.addMutation(m);
}
lastFlushTime = flush(bw, count, flushInterval, lastFlushTime);
+ if (count >= num)
+ break out;
}
}
Modified: accumulo/trunk/test/system/continuous/continuous-env.sh.example
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/continuous-env.sh.example?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/continuous-env.sh.example (original)
+++ accumulo/trunk/test/system/continuous/continuous-env.sh.example Fri Jun 15 20:19:31 2012
@@ -32,6 +32,9 @@ DEBUG_WALKER=off
DEBUG_BATCH_WALKER=off
DEBUG_SCANNER=off
+#the number of entries each client should write
+NUM=9223372036854775807
+
#the minimum random row to generate
MIN=0
Modified: accumulo/trunk/test/system/continuous/start-batchwalkers.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-batchwalkers.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-batchwalkers.sh (original)
+++ accumulo/trunk/test/system/continuous/start-batchwalkers.sh Fri Jun 15 20:19:31 2012
@@ -24,5 +24,5 @@ if [ "$DEBUG_BATCH_WALKER" = "on" ] ; th
DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.log";
fi
-pssh -h batch_walkers.txt "nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousBatchWalker $DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $BATCH_WALKER_SLEEP $BATCH_WALKER_BATCH_SIZE $BATCH_WALKER_THREADS >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.err &" < /dev/null
+pssh -h batch_walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousBatchWalker $DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $BATCH_WALKER_SLEEP $BATCH_WALKER_BATCH_SIZE $BATCH_WALKER_THREADS >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.err &" < /dev/null
Modified: accumulo/trunk/test/system/continuous/start-ingest.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-ingest.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-ingest.sh (original)
+++ accumulo/trunk/test/system/continuous/start-ingest.sh Fri Jun 15 20:19:31 2012
@@ -24,5 +24,5 @@ if [ "$DEBUG_INGEST" = "on" ] ; then
DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.log";
fi
-pssh -h ingesters.txt "nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousIngest $DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $MAX_CF $MAX_CQ $MAX_MEM $MAX_LATENCY $NUM_THREADS $CHECKSUM >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.err &" < /dev/null
+pssh -h ingesters.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousIngest $DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $NUM $MIN $MAX $MAX_CF $MAX_CQ $MAX_MEM $MAX_LATENCY $NUM_THREADS $CHECKSUM >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.err &" < /dev/null
Modified: accumulo/trunk/test/system/continuous/start-scanners.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-scanners.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-scanners.sh (original)
+++ accumulo/trunk/test/system/continuous/start-scanners.sh Fri Jun 15 20:19:31 2012
@@ -24,5 +24,5 @@ if [ "$DEBUG_SCANNER" = "on" ] ; then
DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.log";
fi
-pssh -h scanners.txt "nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousScanner $DEBUG_SCANNERS $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SCANNER_SLEEP_TIME $SCANNER_ENTRIES >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.err &" < /dev/null
+pssh -h scanners.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousScanner $DEBUG_SCANNERS $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SCANNER_SLEEP_TIME $SCANNER_ENTRIES >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.err &" < /dev/null
Modified: accumulo/trunk/test/system/continuous/start-stats.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-stats.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-stats.sh (original)
+++ accumulo/trunk/test/system/continuous/start-stats.sh Fri Jun 15 20:19:31 2012
@@ -18,6 +18,8 @@
. continuous-env.sh
+mkdir -p $CONTINUOUS_LOG_DIR
+
CONFIG_OUT=$CONTINUOUS_LOG_DIR/`date +%Y%m%d%H%M%S`_`hostname`_config.out
cat $ACCUMULO_HOME/conf/accumulo-env.sh > $CONFIG_OUT
Modified: accumulo/trunk/test/system/continuous/start-walkers.sh
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/continuous/start-walkers.sh?rev=1350779&r1=1350778&r2=1350779&view=diff
==============================================================================
--- accumulo/trunk/test/system/continuous/start-walkers.sh (original)
+++ accumulo/trunk/test/system/continuous/start-walkers.sh Fri Jun 15 20:19:31 2012
@@ -24,5 +24,5 @@ if [ "$DEBUG_WALKER" = "on" ] ; then
DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.log";
fi
-pssh -h walkers.txt "nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousWalk $DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SLEEP_TIME >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.err &" < /dev/null
+pssh -h walkers.txt "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.test.continuous.ContinuousWalk $DEBUG_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $MIN $MAX $SLEEP_TIME >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_walk.err &" < /dev/null