You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/01/25 17:24:24 UTC
[5/5] accumulo-testing git commit: ACCUMULO-4510 Refactored Continous
Ingest tests
ACCUMULO-4510 Refactored Continous Ingest tests
* Continuous ingest applications can be launched locally or in YARN and
are now configured by accumulo-testing.properties file
* Applications are now launched using the accumulo-testing command
rather than custom bash scripts
* Removed continuous stats collector as its uses Accumulo
internals and corresponding report generation code.
* Agitator was seperated from continuous ingests. It is
run by accumulo-testing command and configured by
accumulo-testing-env.sh
Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/fc3ddfc4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/fc3ddfc4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/fc3ddfc4
Branch: refs/heads/master
Commit: fc3ddfc4f9a2a259a04d58e87fb97da89a6c5dbc
Parents: b81229d
Author: Mike Walch <mw...@apache.org>
Authored: Thu Jan 12 11:03:41 2017 -0500
Committer: Mike Walch <mw...@apache.org>
Committed: Wed Jan 25 12:19:14 2017 -0500
----------------------------------------------------------------------
.gitignore | 1 +
README.md | 96 +++++++-
bin/accumulo-testing | 163 ++++++++++++-
conf/accumulo-testing-env.sh.example | 28 +++
conf/accumulo-testing.properties.example | 81 ++++++-
continuous/README.md | 103 ---------
continuous/analyze-missing.pl | 127 ----------
continuous/analyze-missing.sh | 23 --
continuous/batch_walkers.txt.example | 16 --
continuous/continuous-env.sh.example | 131 -----------
continuous/datanode-agitator.pl | 140 ------------
continuous/hdfs-agitator.pl | 217 ------------------
continuous/ingesters.txt.example | 17 --
continuous/master-agitator.pl | 92 --------
continuous/report.pl | 120 ----------
continuous/run-moru.sh | 37 ---
continuous/run-verify.sh | 42 ----
continuous/scanners.txt.example | 16 --
continuous/start-agitator.sh | 72 ------
continuous/start-batchwalkers.sh | 42 ----
continuous/start-ingest.sh | 45 ----
continuous/start-scanners.sh | 41 ----
continuous/start-stats.sh | 49 ----
continuous/start-walkers.sh | 41 ----
continuous/stop-agitator.sh | 51 -----
continuous/stop-batchwalkers.sh | 33 ---
continuous/stop-ingest.sh | 33 ---
continuous/stop-scanners.sh | 33 ---
continuous/stop-stats.sh | 33 ---
continuous/stop-walkers.sh | 33 ---
continuous/tserver-agitator.pl | 134 -----------
continuous/walkers.txt.example | 17 --
.../apache/accumulo/testing/core/TestEnv.java | 179 +++++++++++++++
.../apache/accumulo/testing/core/TestProps.java | 94 +++++++-
.../core/continuous/ContinuousBatchWalker.java | 44 ++--
.../testing/core/continuous/ContinuousEnv.java | 66 ++++++
.../core/continuous/ContinuousIngest.java | 105 ++++-----
.../testing/core/continuous/ContinuousMoru.java | 74 +++---
.../core/continuous/ContinuousQuery.java | 74 ------
.../core/continuous/ContinuousScanner.java | 42 ++--
.../continuous/ContinuousStatsCollector.java | 206 -----------------
.../core/continuous/ContinuousVerify.java | 89 +++----
.../testing/core/continuous/ContinuousWalk.java | 91 ++------
.../testing/core/continuous/CreateTable.java | 74 ++++++
.../testing/core/continuous/GenSplits.java | 87 -------
.../testing/core/continuous/HistData.java | 49 ----
.../testing/core/continuous/Histogram.java | 153 -------------
.../core/continuous/PrintScanTimeHistogram.java | 95 --------
.../testing/core/continuous/TimeBinner.java | 2 +-
.../testing/core/randomwalk/Environment.java | 229 -------------------
.../testing/core/randomwalk/Fixture.java | 4 +-
.../testing/core/randomwalk/Framework.java | 11 +-
.../testing/core/randomwalk/Module.java | 8 +-
.../accumulo/testing/core/randomwalk/Node.java | 2 +-
.../testing/core/randomwalk/RandWalkEnv.java | 84 +++++++
.../core/randomwalk/bulk/BulkImportTest.java | 6 +-
.../core/randomwalk/bulk/BulkMinusOne.java | 4 +-
.../core/randomwalk/bulk/BulkPlusOne.java | 8 +-
.../testing/core/randomwalk/bulk/BulkTest.java | 6 +-
.../testing/core/randomwalk/bulk/Compact.java | 6 +-
.../core/randomwalk/bulk/ConsistencyCheck.java | 10 +-
.../testing/core/randomwalk/bulk/Merge.java | 6 +-
.../core/randomwalk/bulk/SelectiveBulkTest.java | 4 +-
.../core/randomwalk/bulk/SelectiveQueueing.java | 6 +-
.../testing/core/randomwalk/bulk/Setup.java | 6 +-
.../testing/core/randomwalk/bulk/Split.java | 6 +-
.../testing/core/randomwalk/bulk/Verify.java | 12 +-
.../core/randomwalk/concurrent/AddSplits.java | 6 +-
.../core/randomwalk/concurrent/BatchScan.java | 6 +-
.../core/randomwalk/concurrent/BatchWrite.java | 6 +-
.../core/randomwalk/concurrent/BulkImport.java | 7 +-
.../concurrent/ChangeAuthorizations.java | 6 +-
.../concurrent/ChangePermissions.java | 6 +-
.../randomwalk/concurrent/CheckPermission.java | 6 +-
.../core/randomwalk/concurrent/CloneTable.java | 6 +-
.../core/randomwalk/concurrent/Compact.java | 6 +-
.../concurrent/ConcurrentFixture.java | 6 +-
.../core/randomwalk/concurrent/Config.java | 30 +--
.../randomwalk/concurrent/CreateNamespace.java | 6 +-
.../core/randomwalk/concurrent/CreateTable.java | 6 +-
.../core/randomwalk/concurrent/CreateUser.java | 6 +-
.../randomwalk/concurrent/DeleteNamespace.java | 6 +-
.../core/randomwalk/concurrent/DeleteRange.java | 6 +-
.../core/randomwalk/concurrent/DeleteTable.java | 6 +-
.../core/randomwalk/concurrent/DropUser.java | 6 +-
.../randomwalk/concurrent/IsolatedScan.java | 6 +-
.../core/randomwalk/concurrent/ListSplits.java | 6 +-
.../core/randomwalk/concurrent/Merge.java | 6 +-
.../randomwalk/concurrent/OfflineTable.java | 6 +-
.../randomwalk/concurrent/RenameNamespace.java | 6 +-
.../core/randomwalk/concurrent/RenameTable.java | 6 +-
.../core/randomwalk/concurrent/Replication.java | 10 +-
.../core/randomwalk/concurrent/ScanTable.java | 6 +-
.../core/randomwalk/concurrent/Setup.java | 4 +-
.../core/randomwalk/conditional/Compact.java | 6 +-
.../core/randomwalk/conditional/Flush.java | 6 +-
.../core/randomwalk/conditional/Init.java | 6 +-
.../core/randomwalk/conditional/Merge.java | 6 +-
.../core/randomwalk/conditional/Setup.java | 10 +-
.../core/randomwalk/conditional/Split.java | 6 +-
.../core/randomwalk/conditional/TearDown.java | 4 +-
.../core/randomwalk/conditional/Transfer.java | 6 +-
.../core/randomwalk/conditional/Verify.java | 6 +-
.../testing/core/randomwalk/image/Commit.java | 4 +-
.../core/randomwalk/image/ImageFixture.java | 12 +-
.../testing/core/randomwalk/image/ScanMeta.java | 6 +-
.../testing/core/randomwalk/image/TableOp.java | 6 +-
.../testing/core/randomwalk/image/Verify.java | 6 +-
.../testing/core/randomwalk/image/Write.java | 4 +-
.../core/randomwalk/multitable/Commit.java | 4 +-
.../core/randomwalk/multitable/CopyTable.java | 20 +-
.../core/randomwalk/multitable/CreateTable.java | 8 +-
.../core/randomwalk/multitable/DropTable.java | 6 +-
.../multitable/MultiTableFixture.java | 8 +-
.../randomwalk/multitable/OfflineTable.java | 8 +-
.../core/randomwalk/multitable/Write.java | 4 +-
.../randomwalk/security/AlterSystemPerm.java | 6 +-
.../core/randomwalk/security/AlterTable.java | 8 +-
.../randomwalk/security/AlterTablePerm.java | 16 +-
.../core/randomwalk/security/Authenticate.java | 10 +-
.../core/randomwalk/security/ChangePass.java | 8 +-
.../core/randomwalk/security/CreateTable.java | 8 +-
.../core/randomwalk/security/CreateUser.java | 8 +-
.../core/randomwalk/security/DropTable.java | 14 +-
.../core/randomwalk/security/DropUser.java | 8 +-
.../randomwalk/security/SecurityFixture.java | 10 +-
.../core/randomwalk/security/SetAuths.java | 10 +-
.../core/randomwalk/security/TableOp.java | 6 +-
.../core/randomwalk/security/Validate.java | 12 +-
.../randomwalk/security/WalkingSecurity.java | 12 +-
.../core/randomwalk/sequential/BatchVerify.java | 6 +-
.../core/randomwalk/sequential/Commit.java | 4 +-
.../randomwalk/sequential/MapRedVerify.java | 16 +-
.../sequential/SequentialFixture.java | 12 +-
.../core/randomwalk/sequential/Write.java | 4 +-
.../core/randomwalk/shard/BulkInsert.java | 13 +-
.../core/randomwalk/shard/CloneIndex.java | 8 +-
.../testing/core/randomwalk/shard/Commit.java | 4 +-
.../core/randomwalk/shard/CompactFilter.java | 10 +-
.../testing/core/randomwalk/shard/Delete.java | 4 +-
.../core/randomwalk/shard/DeleteSomeDocs.java | 8 +-
.../core/randomwalk/shard/DeleteWord.java | 8 +-
.../core/randomwalk/shard/ExportIndex.java | 29 ++-
.../testing/core/randomwalk/shard/Flush.java | 6 +-
.../testing/core/randomwalk/shard/Grep.java | 8 +-
.../testing/core/randomwalk/shard/Insert.java | 4 +-
.../testing/core/randomwalk/shard/Merge.java | 12 +-
.../testing/core/randomwalk/shard/Reindex.java | 8 +-
.../testing/core/randomwalk/shard/Search.java | 10 +-
.../core/randomwalk/shard/ShardFixture.java | 14 +-
.../testing/core/randomwalk/shard/Split.java | 6 +-
.../core/randomwalk/shard/VerifyIndex.java | 12 +-
.../core/randomwalk/unit/CreateTable.java | 4 +-
.../core/randomwalk/unit/DeleteTable.java | 4 +-
.../testing/core/randomwalk/unit/Ingest.java | 4 +-
.../testing/core/randomwalk/unit/Scan.java | 4 +-
.../testing/core/randomwalk/unit/Verify.java | 4 +-
.../randomwalk/ReplicationRandomWalkIT.java | 8 +-
libexec/analyze-missing.pl | 127 ++++++++++
libexec/datanode-agitator.pl | 140 ++++++++++++
libexec/hdfs-agitator.pl | 217 ++++++++++++++++++
libexec/master-agitator.pl | 92 ++++++++
libexec/tserver-agitator.pl | 134 +++++++++++
163 files changed, 2124 insertions(+), 3346 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index f534230..7e54b72 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@
/target/
/*.iml
/.idea
+/logs/
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index f19b91b..0d2b538 100644
--- a/README.md
+++ b/README.md
@@ -32,9 +32,8 @@ on your machine as well as an Accumulo instance to use for testing.
The random walk test generates client behavior on an Apache Accumulo instance by randomly walking a
graph of client operations.
-Before running random walk, review the `test.common.*` and `test.randomwalk.*` properties in
-`accumulo-testing.properties` file. A test module must also be specified. See the [modules][modules]
-directory for a list of available ones.
+Before running random walk, review the `test.common.*` properties in `accumulo-testing.properties`
+file. A test module must also be specified. See the [modules] directory for a list of available ones.
The command below will start a single random walker in a local process using the [Image.xml][image]
module.
@@ -46,9 +45,94 @@ walkers in 5 containers in YARN using the Image.xml module.
./bin/accumulo-testing rw-yarn 5 Image.xml
-This command will create an application in YARN and exit when all containers for the test have started.
-While its running, you can view logs for each random walker using the YARN resource manager. The YARN
-application can be killed at any time using the YARN resource manager or command line tool.
+This command will create an application in YARN and exit when all containers for the test have
+started. While its running, you can view logs for each random walker using the YARN resource manager.
+The YARN application can be killed at any time using the YARN resource manager or command line tool.
+
+## Continuous Ingest & Query
+
+The Continuous Ingest test runs many ingest clients that continually create linked lists of data
+in Accumulo. During ingest, query applications can be run to continously walk and verify the the
+linked lists and put a query load on Accumulo. At some point, the ingest clients are stopped and
+a MapReduce job is run to ensure that there are no holes in any linked list.
+
+The nodes in the linked list are random. This causes each linked list to spread across the table.
+Therefore, if one part of a table loses data, then it will be detected by references in another
+part of table.
+
+Before running any of the Continuous Ingest applications, make sure that the
+`accumulo-testing.properties` file exists in `conf/` and review all properties with the
+`test.ci.*` prefix.
+
+First, run the command below to create an Accumulo table for the continuous ingest tests. The name of the
+table is set by the property `test.ci.common.accumulo.table` (its value defaults to `ci`) in the file
+`accumulo-testing.properties`:
+
+ ./bin/accumulo-testing ci-createtable
+
+The continuous ingest tests have several applications that can either be started in a local process
+or run in multiple containers across a cluster using YARN. The `ci-local` command starts a local
+application which will run continuously until you stop using `ctrl-c`:
+
+ ./bin/accumulo-testing ci-local <application>
+
+The `ci-yarn` command starts an application in `<num>` containers in YARN. All containers will run
+continuously performing the same work until you kill the application in YARN. The logs for the
+application can be viewed using the YARN resource manager.
+
+ ./bin/accumulo-testing ci-yarn <num> <application>
+
+Below is a list of available continuous ingest applications. You should run the `ingest` application
+first to add data to your table.
+
+* `ingest` - Inserts data into Accumulo that will form a random graph.
+* `walk` - Randomly walks the graph created by ingest application using scanner. Each walker
+ produces detailed statistics on query/scan times.
+* `batchwalk` - Randomly walks the graph created by ingest using a batch scanner.
+* `scan` - Scans the graph
+
+The continuous ingest test has two MapReduce jobs that are used to verify and stress
+Accumulo and have the following command:
+
+ ./bin/accumulo-testing ci-mapred <application>
+
+Below is a list of available MapReduce applications:
+
+1. `verify` - Runs a MapReduce job that verifies all data created by continuous ingest. Before
+running, review all `test.ci.verify.*` properties. Do not run ingest while running this command as
+it will cause erroneous reporting of UNDEFINED nodes. Each entry, except for the first batch of
+entries, inserted by continuous ingest references a previously flushed entry. Since we are
+referencing flushed entries, they should always exist. The MapReduce job checks that all referenced
+entries exist. If it finds any that do not exist it will increment the UNDEFINED counter and emit
+the referenced but undefined node. The MapReduce job produces two other counts: REFERENCED and
+UNREFERENCED. It is expected that these two counts are non zero. REFERENCED counts nodes that are
+defined and referenced. UNREFERENCED counts nodes that defined and unreferenced, these are the
+latest nodes inserted.
+
+2. `moru` - Runs a MapReduce job that stresses Accumulo by reading and writing the continuous ingest
+table. This MapReduce job will write out an entry for every entry in the table (except for ones
+created by the MapReduce job itself). Stop ingest before running this MapReduce job. Do not run more
+than one instance of this MapReduce job concurrently against a table.
+
+## Agitator
+
+The agitator will periodically kill the Accumulo master, tablet server, and Hadoop data node
+processes on random nodes. Before running the agitator you should create `accumulo-testing-env.sh`
+in `conf/` and review all of the agitator settings. The command below will start the agitator:
+
+ ./bin/accumulo-testing agitator start
+
+You can run this script as root and it will properly start processes as the user you configured in
+`accumulo-testing-env.sh` (`AGTR_HDFS_USER` for the data node and `AGTR_ACCUMULO_USER` for Accumulo
+processes). If you run it as yourself and the `AGTR_HDFS_USER` and `AGTR_ACCUMULO_USER` values are
+the same as your user, the agitator will not change users. In the case where you run the agitator as
+a non-privileged user which isn't the same as `AGTR_HDFS_USER` or `AGTR_ACCUMULO_USER`, the agitator
+will attempt to `sudo` to these users, which relies on correct configuration of sudo. Also, be sure
+that your `AGTR_HDFS_USER` has password-less `ssh` configured.
+
+Run the command below stop the agitator:
+
+ ./bin/accumulo-testing agitator stop
[modules]: core/src/main/resources/randomwalk/modules
[image]: core/src/main/resources/randomwalk/modules/Image.xml
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/bin/accumulo-testing
----------------------------------------------------------------------
diff --git a/bin/accumulo-testing b/bin/accumulo-testing
index dc6f5da..7cf7206 100755
--- a/bin/accumulo-testing
+++ b/bin/accumulo-testing
@@ -25,8 +25,16 @@ function print_usage() {
Usage: accumulo-testing <command> (<argument>)
Possible commands:
- rw-local <module> Runs randomwalk <module> in local java process
- rw-yarn <num> <module> Runs randomwalk <module> in <num> containers on YARN
+ agitator <command> Runs agitator <command>. Available commands: start, stop
+ ci-createtable Creates Accumulo table with splits for continuous ingest tests
+ ci-local <application> Runs continuous ingest <application> in local java process
+ Applications: ingest, walk, batchwalk, scan
+ ci-yarn <num> <application> Runs continuous ingest <application> in <num> containers in YARN
+ Applications: ingest, walk, batchwalk, scan
+ ci-mapred <application> Runs continuous ingest mapreduce <application>
+ Applications: verify, moru
+ rw-local <module> Runs randomwalk <module> in local java process
+ rw-yarn <num> <module> Runs randomwalk <module> in <num> containers on YARN
EOF
}
@@ -69,8 +77,143 @@ function build_shade_jar() {
fi
}
+function determine_app_main() {
+ ci_package="org.apache.accumulo.testing.core.continuous"
+ case "$1" in
+ ingest)
+ ci_main="${ci_package}.ContinuousIngest"
+ ;;
+ walk)
+ ci_main="${ci_package}.ContinuousWalk"
+ ;;
+ batchwalk)
+ ci_main="${ci_package}.ContinuousBatchWalker"
+ ;;
+ scan)
+ ci_main="${ci_package}.ContinuousScanner"
+ ;;
+ *)
+ echo "Unknown application: $1"
+ print_usage
+ exit 1
+ esac
+}
+
+function determine_mapred_main() {
+ ci_package="org.apache.accumulo.testing.core.continuous"
+ case "$1" in
+ verify)
+ ci_main="${ci_package}.ContinuousVerify"
+ ;;
+ moru)
+ ci_main="${ci_package}.ContinuousMoru"
+ ;;
+ *)
+ echo "Unknown application: $1"
+ print_usage
+ exit 1
+ esac
+}
+
+function start_agitator() {
+ mkdir -p "${at_home}/logs"
+ log_base="${at_home}/logs/$(date +%Y%m%d%H%M%S)_$(hostname)"
+ libexec="${at_home}/libexec"
+ master_log="${log_base}_master-agitator"
+ tserver_log="${log_base}_tserver-agitator"
+ datanode_log="${log_base}_datanode-agitator"
+ master_cmd="nohup ${libexec}/master-agitator.pl $AGTR_MASTER_KILL_SLEEP_TIME $AGTR_MASTER_RESTART_SLEEP_TIME"
+ tserver_cmd="nohup ${libexec}/tserver-agitator.pl $AGTR_TSERVER_KILL_SLEEP_TIME $AGTR_TSERVER_RESTART_SLEEP_TIME $AGTR_TSERVER_MIN_KILL $AGTR_TSERVER_MAX_KILL"
+ datanode_cmd="nohup ${libexec}/datanode-agitator.pl $AGTR_DATANODE_KILL_SLEEP_TIME $AGTR_DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $AGTR_DATANODE_MIN_KILL $AGTR_DATANODE_MAX_KILL"
+ [[ -n $AGITATOR_USER ]] || AGITATOR_USER=$(whoami)
+
+ if [[ $AGITATOR_USER == root ]]; then
+ echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER using su. Running datanode-agitator as $AGTR_HDFS_USER using su."
+ su -c "$master_cmd >${master_log}.out 2>${master_log}.err" & -m - "$AGTR_ACCUMULO_USER"
+ su -c "$tserver_cmd >${tserver_log}.out 2>${tserver_log}.err" & -m - "$AGTR_ACCUMULO_USER"
+ su -c "$datanode_cmd >${datanode_log}.out 2>${datanode_log}.err" & -m - "$AGTR_HDFS_USER"
+ elif [[ $AGITATOR_USER == "$AGTR_ACCUMULO_USER" ]]; then
+ echo "Running master-agitator and tserver-agitator as $AGITATOR_USER Running datanode-agitator as $AGTR_HDFS_USER using sudo."
+ $master_cmd > "${master_log}.out" 2> "${master_log}.err" &
+ $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
+ sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> "${datanode_log}.err" &
+ else
+ echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER using sudo. Running datanode-agitator as $AGTR_HDFS_USER using sudo."
+ # Not root, and not the accumulo user, hope you can sudo to it
+ sudo -u "$AGTR_ACCUMULO_USER" "$master_cmd" > "${master_log}.out" 2> "${master_log}.err" &
+ sudo -u "$AGTR_ACCUMULO_USER" "$tserver_cmd" > "${tserver_log}.out" 2> "${tserver_log}.err" &
+ sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> "${datanode_log}.err" & -m - "$AGTR_HDFS_USER"
+ fi
+
+ if ${AGTR_HDFS:-false} ; then
+ agitator_log=${log_base}_hdfs-agitator
+ sudo -u "$AGTR_HDFS_SUPERUSER" nohup "${libexec}/hdfs-agitator.pl" --sleep "${AGTR_HDFS_SLEEP_TIME}" --hdfs-cmd "${AGTR_HDFS_COMMAND}" --superuser "${AGTR_HDFS_SUPERUSER}" >"${agitator_log}.out" 2>"${agitator_log}.err" &
+ fi
+}
+
+function stop_agitator() {
+ [[ -n $AGITATOR_USER ]] || AGITATOR_USER=$(whoami)
+ if [[ $AGITATOR_USER == root ]]; then
+ echo "Stopping all processes matching 'agitator.pl' as root"
+ pkill -f agitator.pl 2>/dev/null
+ elif [[ $AGITATOR_USER == "$AGTR_ACCUMULO_USER" ]]; then
+ echo "Stopping all processes matching 'datanode-agitator.pl' as $AGTR_HDFS_USER"
+ sudo -u "$AGTR_HDFS_USER" pkill -f datanode-agitator.pl 2>/dev/null
+ echo "Stopping all processes matching 'hdfs-agitator.pl' as $AGTR_HDFS_USER"
+ sudo -u "$AGTR_HDFS_USER" pkill -f hdfs-agitator.pl 2>/dev/null
+ echo "Stopping all processes matching 'agitator.pl' as $AGITATOR_USER"
+ pkill -f agitator.pl 2>/dev/null 2>/dev/null
+ else
+ echo "Stopping all processes matching 'datanode-agitator.pl' as $AGTR_HDFS_USER"
+ sudo -u "$AGTR_HDFS_USER" pkill -f datanode-agitator.pl 2>/dev/null
+ echo "Stopping all processes matching 'hdfs-agitator.pl' as $AGTR_HDFS_USER"
+ sudo -u "$AGTR_HDFS_USER" pkill -f hdfs-agitator.pl 2>/dev/null
+ echo "Stopping all processes matching 'agitator.pl' as $AGTR_ACCUMULO_USER"
+ sudo -u "$AGTR_ACCUMULO_USER" pkill -f agitator.pl 2>/dev/null
+ fi
+}
+
randomwalk_main="org.apache.accumulo.testing.core.randomwalk.Framework"
case "$1" in
+ci-createtable)
+ build_shade_jar
+ java -Dlog4j.configuration="file:$log4j_config" -cp "$at_shaded_jar" org.apache.accumulo.testing.core.continuous.CreateTable "$at_props"
+ ;;
+ci-local)
+ if [ -z "$2" ]; then
+ echo "ERROR: <application> needs to be set"
+ print_usage
+ exit 1
+ fi
+ determine_app_main "$2"
+ build_shade_jar
+ java -Dlog4j.configuration="file:$log4j_config" -cp "$at_shaded_jar" "$ci_main" "$at_props"
+ ;;
+ci-yarn)
+ if [ -z "$2" ]; then
+ echo "ERROR: <num> needs to be set"
+ print_usage
+ exit 1
+ fi
+ if [ -z "$3" ]; then
+ echo "ERROR: <application> needs to be set"
+ print_usage
+ exit 1
+ fi
+ determine_app_main "$3"
+ build_shade_jar
+ mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloCITest-$3 -j $at_shaded_jar -m $ci_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties"
+ ;;
+ci-mapred)
+ if [ -z "$2" ]; then
+ echo "ERROR: <application> needs to be set"
+ print_usage
+ exit 1
+ fi
+ determine_mapred_main "$2"
+ build_shade_jar
+ "$HADOOP_PREFIX"/bin/yarn jar "$at_shaded_jar" "$ci_main" "$at_props"
+ ;;
rw-local)
if [ -z "$2" ]; then
echo "ERROR: <module> needs to be set"
@@ -92,7 +235,21 @@ rw-yarn)
exit 1
fi
build_shade_jar
- mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloRandomWalkTest -j $at_shaded_jar -m $randomwalk_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties $3"
+ mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloRWTest-$3 -j $at_shaded_jar -m $randomwalk_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties $3"
+ ;;
+agitator)
+ case "$2" in
+ start)
+ start_agitator
+ ;;
+ stop)
+ stop_agitator
+ ;;
+ *)
+ echo "ERROR: unknown command - $2"
+ print_usage
+ exit 1
+ esac
;;
*)
echo "Unknown command: $1"
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/conf/accumulo-testing-env.sh.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example
index 2e3554e..3f71463 100644
--- a/conf/accumulo-testing-env.sh.example
+++ b/conf/accumulo-testing-env.sh.example
@@ -24,3 +24,31 @@ test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR=/path/to/hadoop/etc/hadoop
export ACCUMULO_VERSION=`accumulo version`
export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'`
export ZOOKEEPER_VERSION=3.4.9
+
+# Agitator
+# ========
+# Accumulo user
+AGTR_ACCUMULO_USER=$(whoami)
+# Time (in minutes) between killing Accumulo masters
+AGTR_MASTER_KILL_SLEEP_TIME=60
+AGTR_MASTER_RESTART_SLEEP_TIME=2
+# Time (in minutes) between killing Accumulo tservers
+AGTR_TSERVER_KILL_SLEEP_TIME=20
+AGTR_TSERVER_RESTART_SLEEP_TIME=10
+# Min and max number of Accumulo tservers that the agitator will kill at once
+AGTR_TSERVER_MIN_KILL=1
+AGTR_TSERVER_MAX_KILL=1
+# Amount of time (in minutes) the agitator should sleep before killing datanodes
+AGTR_DATANODE_KILL_SLEEP_TIME=20
+# Amount of time (in minutes) the agitator should wait before restarting datanodes
+AGTR_DATANODE_RESTART_SLEEP_TIME=10
+# Min and max number of datanodes the agitator will kill at once
+AGTR_DATANODE_MIN_KILL=1
+AGTR_DATANODE_MAX_KILL=1
+# HDFS agitation
+AGTR_HDFS_USER=$(whoami)
+AGTR_HDFS=false
+AGTR_HDFS_SLEEP_TIME=10
+AGTR_HDFS_SUPERUSER=hdfs
+AGTR_HDFS_COMMAND="${HADOOP_PREFIX:-/usr/lib/hadoop}/share/hadoop/hdfs/bin/hdfs"
+AGTR_HDFS_SUDO=$(which sudo)
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/conf/accumulo-testing.properties.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing.properties.example b/conf/accumulo-testing.properties.example
index 05cbaf5..1d43e12 100644
--- a/conf/accumulo-testing.properties.example
+++ b/conf/accumulo-testing.properties.example
@@ -31,14 +31,77 @@ test.common.zookeepers=localhost:2181
test.common.yarn.container.memory.mb=1024
# Number of cores given to each container (if running in YARN)
test.common.yarn.container.cores=1
+# Max memory (in bytes) each batch writer will use to buffer writes
+test.common.bw.max.memory.bytes=100000000
+# Max latency (in milliseconds) that each batch writer will buffer data
+test.common.bw.max.latency.ms=600000
+# Number of write thread for each batch writer
+test.common.bw.num.threads=4
+# Number of threads used by batch scanner
+test.common.bs.num.threads=8
+# Number of key/value entries to pull during scan
+test.common.scanner.batch.size=1000
-########################
-# Random walk properties
-########################
+###################################
+# Continuous ingest test properties
+###################################
-# Max memory for multi-table batch writer
-test.randomwalk.bw.max.mem=100000000
-# Max latency in milliseconds for multi-table batch writer
-test.randomwalk.bw.max.latency=600000
-# Number of write thread for multi-table batch writer
-test.randomwalk.bw.num.threads=4
+# Common
+# ------
+# Accumulo table used by continuous tests
+test.ci.common.accumulo.table=ci
+# Number of tablets that should exist in Accumulo table when created
+test.ci.common.accumulo.num.tablets=20
+# Optional authorizations that if specified will be randomly selected by scanners and walkers
+# Format: a,b|a,b,c|c
+test.ci.common.auths=
+
+# Ingest
+# ------
+# Number of entries each ingest client should write
+test.ci.ingest.client.entries=9223372036854775807
+# Minimum random row to generate
+test.ci.ingest.row.min=0
+# Maximum random row to generate
+test.ci.ingest.row.max=9223372036854775807
+# Maximum number of random column families to generate
+test.ci.ingest.max.cf=32767
+# Maximum number of random column qualifiers to generate
+test.ci.ingest.max.cq=32767
+# Optional visibilities (in CSV format) that if specified will be randomly selected by ingesters for
+# each linked list
+test.ci.ingest.visibilities=
+# Checksums will be generated during ingest if set to true
+test.ci.ingest.checksum=true
+
+# Batch walker
+# ------------
+# Sleep time between batch scans (in ms)
+test.ci.batch.walker.sleep.ms=10000
+# Scan batch size
+test.ci.batch.walker.batch.size=10000
+
+# Walker
+# ------
+# Sleep time between scans (in ms)
+test.ci.walker.sleep.ms=10000
+
+# Scanner
+# -------
+# Sleep time between scans (in ms)
+test.ci.scanner.sleep.ms=10000
+# Scanner entries
+test.ci.scanner.entries=5000
+
+# Verify
+# -----
+# Maximum number of mapreduce mappers
+test.ci.verify.max.maps=64
+# Number of mapreduce reducers
+test.ci.verify.reducers=64
+# Perform the verification directly on the files while the table is offline
+test.ci.verify.scan.offline=false
+# Comma separated list of auths to use for verify
+test.ci.verify.auths=
+# Location in HDFS to store output. Must not exist.
+test.ci.verify.output.dir=/tmp/ci-verify
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/README.md
----------------------------------------------------------------------
diff --git a/continuous/README.md b/continuous/README.md
deleted file mode 100644
index 31ee4bd..0000000
--- a/continuous/README.md
+++ /dev/null
@@ -1,103 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-Continuous Query and Ingest
-===========================
-
-This directory contains a suite of scripts for placing continuous query and
-ingest load on accumulo. The purpose of these script is two-fold. First,
-place continuous load on accumulo to see if breaks. Second, collect
-statistics in order to understand how accumulo behaves. To run these scripts
-copy all of the `.example` files and modify them. You can put these scripts in
-the current directory or define a `CONTINUOUS_CONF_DIR` where the files will be
-read from. These scripts rely on `pssh`. Before running any script you may need
-to use `pssh` to create the log directory on each machine (if you want it local).
-Also, create the table "ci" before running. You can run
-`org.apache.accumulo.test.continuous.GenSplits` to generate splits points for a
-continuous ingest table.
-
-The following ingest scripts insert data into accumulo that will form a random
-graph.
-
-> $ start-ingest.sh
-> $ stop-ingest.sh
-
-The following query scripts randomly walk the graph created by the ingesters.
-Each walker produce detailed statistics on query/scan times.
-
-> $ start-walkers.sh
-> $ stop-walker.sh
-
-The following scripts start and stop batch walkers.
-
-> $ start-batchwalkers.sh
-> $ stop-batchwalkers.sh
-
-And the following scripts start and stop scanners.
-
-> $ start-scanners.sh
-> $ stop-scanners.sh
-
-In addition to placing continuous load, the following scripts start and stop a
-service that continually collect statistics about accumulo and HDFS.
-
-> $ start-stats.sh
-> $ stop-stats.sh
-
-Optionally, start the agitator to periodically kill the tabletserver and/or datanode
-process(es) on random nodes. You can run this script as root and it will properly start
-processes as the user you configured in `continuous-env.sh` (`HDFS_USER` for the Datanode and
-`ACCUMULO_USER` for Accumulo processes). If you run it as yourself and the `HDFS_USER` and
-`ACCUMULO_USER` values are the same as your user, the agitator will not change users. In
-the case where you run the agitator as a non-privileged user which isn't the same as `HDFS_USER`
-or `ACCUMULO_USER`, the agitator will attempt to `sudo` to these users, which relies on correct
-configuration of sudo. Also, be sure that your `HDFS_USER` has password-less `ssh` configured.
-
-> $ start-agitator.sh
-> $ stop-agitator.sh
-
-Start all three of these services and let them run for a few hours. Then run
-`report.pl` to generate a simple HTML report containing plots and histograms
-showing what has transpired.
-
-A MapReduce job to verify all data created by continuous ingest can be run
-with the following command. Before running the command modify the `VERIFY_*`
-variables in `continuous-env.sh` if needed. Do not run ingest while running this
-command, this will cause erroneous reporting of UNDEFINED nodes. The MapReduce
-job will scan a reference after it has scanned the definition.
-
-> $ run-verify.sh
-
-Each entry, except for the first batch of entries, inserted by continuous
-ingest references a previously flushed entry. Since we are referencing flushed
-entries, they should always exist. The MapReduce job checks that all
-referenced entries exist. If it finds any that do not exist it will increment
-the UNDEFINED counter and emit the referenced but undefined node. The MapReduce
-job produces two other counts : REFERENCED and UNREFERENCED. It is
-expected that these two counts are non zero. REFERENCED counts nodes that are
-defined and referenced. UNREFERENCED counts nodes that defined and
-unreferenced, these are the latest nodes inserted.
-
-To stress accumulo, run the following script which starts a MapReduce job
-that reads and writes to your continuous ingest table. This MapReduce job
-will write out an entry for every entry in the table (except for ones created
-by the MapReduce job itself). Stop ingest before running this MapReduce job.
-Do not run more than one instance of this MapReduce job concurrently against a
-table.
-
-> $ run-moru.sh
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/analyze-missing.pl
----------------------------------------------------------------------
diff --git a/continuous/analyze-missing.pl b/continuous/analyze-missing.pl
deleted file mode 100755
index 5cce1b1..0000000
--- a/continuous/analyze-missing.pl
+++ /dev/null
@@ -1,127 +0,0 @@
-#! /usr/bin/env perl
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-use POSIX qw(strftime);
-
-if(scalar(@ARGV) != 4){
- print "Usage : analyze-missing.pl <accumulo home> <continuous log dir> <user> <pass> \n";
- exit(1);
-}
-
-$ACCUMULO_HOME=$ARGV[0];
-$CONTINUOUS_LOG_DIR=$ARGV[1];
-$USER=$ARGV[2];
-$PASS=$ARGV[3];
-
-
-@missing = `grep MIS $CONTINUOUS_LOG_DIR/*.err`;
-
-
-
-for $miss (@missing) {
- chomp($miss);
- ($file, $type, $time, $row) = split(/[: ]/, $miss);
-
- substr($file, -3, 3, "out");
-
- $prevRowLine = `grep -B 1 $row $file | grep SRQ | grep -v $row`;
-
- @prla = split(/\s+/, $prevRowLine);
- $prevRow = $prla[2];
-# print $prevRow."\n";
-
- $aScript = `mktemp /tmp/miss_script.XXXXXXXXXX`;
- chomp($aScript);
- open(AS, ">$aScript") || die;
-
- print AS "table ci\n";
- print AS "scan -b $prevRow -e $prevRow\n";
- print AS "scan -b $row -e $row\n";
- print AS "quit\n";
- close(AS);
-
- $exist = 0;
- $ingestIDSame = 0;
- $ingestId = "";
- $count = 0;
-
- @entries = `$ACCUMULO_HOME/bin/accumulo shell -u $USER -p $PASS -f $aScript | grep $row`;
- system("rm $aScript");
-
- for $entry (@entries){
- chomp($entry);
- @entryA = split(/[: ]+/, $entry);
- if($entryA[0] eq $row){
- $exist = 1;
-
- if($entryA[4] eq $ingestId){
- $ingestIDSame = 1;
- }
- }else{
- $ingestId = $entryA[4];
- $count = hex($entryA[5]);
- }
- }
-
-
- #look in ingest logs
- @ingestLogs = `ls $CONTINUOUS_LOG_DIR/*ingest*.out`;
- @flushTimes = ();
- chomp(@ingestLogs);
- for $ingestLog (@ingestLogs){
- open(IL, "<$ingestLog") || die;
-
-
- while($firstLine = <IL>){
- chomp($firstLine);
- if($firstLine =~ /UUID.*/){
- last;
- }
- }
-
- @iinfo = split(/\s+/,$firstLine);
- if($iinfo[2] eq $ingestId){
- while($line = <IL>){
- if($line =~ /FLUSH (\d+) \d+ \d+ (\d+) \d+/){
- push(@flushTimes, $1);
- if(scalar(@flushTimes) > 3){
- shift(@flushTimes);
- }
- if($count < $2){
- last;
- }
- }
- }
- }
-
-
-
- close(IL);
-
- if(scalar(@flushTimes) > 0){
- last;
- }
- }
-
- $its0 = strftime "%m/%d/%Y_%H:%M:%S", gmtime($flushTimes[0]/1000);
- $its1 = strftime "%m/%d/%Y_%H:%M:%S", gmtime($flushTimes[1]/1000);
- $mts = strftime "%m/%d/%Y_%H:%M:%S", gmtime($time/1000);
-
- print "$row $exist $ingestIDSame $prevRow $ingestId $its0 $its1 $mts\n";
-}
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/analyze-missing.sh
----------------------------------------------------------------------
diff --git a/continuous/analyze-missing.sh b/continuous/analyze-missing.sh
deleted file mode 100755
index e2cfbb1..0000000
--- a/continuous/analyze-missing.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-$ACCUMULO_HOME/test/system/continuous/}
-. "$CONTINUOUS_CONF_DIR/continuous-env.sh"
-
-./analyze-missing.pl "$ACCUMULO_HOME" "$CONTINUOUS_LOG_DIR" "$USER" "$PASS"
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/batch_walkers.txt.example
----------------------------------------------------------------------
diff --git a/continuous/batch_walkers.txt.example b/continuous/batch_walkers.txt.example
deleted file mode 100644
index 63fb8bb..0000000
--- a/continuous/batch_walkers.txt.example
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-localhost
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/continuous-env.sh.example
----------------------------------------------------------------------
diff --git a/continuous/continuous-env.sh.example b/continuous/continuous-env.sh.example
deleted file mode 100644
index 0abd8c3..0000000
--- a/continuous/continuous-env.sh.example
+++ /dev/null
@@ -1,131 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# set this to an absolute path that exist on every machine
-# Inherit values from environment if they are already set.
-HADOOP_HOME=${HADOOP_HOME:-/opt/hadoop}
-HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_HOME}
-ACCUMULO_HOME=${ACCUMULO_HOME:-/opt/accumulo}
-ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-$ACCUMULO_HOME/conf}
-JAVA_HOME=${JAVA_HOME:-/opt/java}
-ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/opt/zookeeper}
-
-CONTINUOUS_LOG_DIR=$ACCUMULO_HOME/test/system/continuous/logs
-INSTANCE_NAME=instance
-ZOO_KEEPERS=zhost1,zhost2
-ACCUMULO_USER=$(whoami)
-HDFS_USER=$(whoami)
-USER=user
-PASS=pass
-TABLE=ci
-
-#set debug to on to enable logging of accumulo client debugging
-DEBUG_INGEST=off
-DEBUG_WALKER=off
-DEBUG_BATCH_WALKER=off
-DEBUG_SCANNER=off
-
-#the number of entries each client should write
-NUM=9223372036854775807
-
-#the minimum random row to generate
-MIN=0
-
-#the maximum random row to generate
-MAX=9223372036854775807
-
-#the maximum number of random column families to generate
-MAX_CF=32767
-
-#the maximum number of random column qualifiers to generate
-MAX_CQ=32767
-
-#an optional file in hdfs containing visibilites. If left blank, then column
-#visibility will not be set. If specified then a random line will be selected
-#from the file and used for column visibility for each linked list.
-VISIBILITIES=''
-
-#the max memory (in bytes) each ingester will use to buffer writes
-MAX_MEM=100000000
-
-#the maximum time (in millis) each ingester will buffer data
-MAX_LATENCY=600000
-
-#the number of threads each ingester will use to write data
-NUM_THREADS=4
-
-#the amount of time (in millis) to sleep between each query
-SLEEP_TIME=10
-
-#an optional file in hdfs containing line of comma seperated auths. If
-#specified, walkers will randomly select lines from this file and use that to
-#set auths.
-AUTHS=''
-
-#determines if checksum are generated, may want to turn of when performance testing
-CHECKSUM=true
-
-#the amount of time (in minutes) the agitator should sleep before killing tservers
-TSERVER_KILL_SLEEP_TIME=20
-
-#the amount of time (in minutes) the agitator should sleep after killing
-# before restarting tservers
-TSERVER_RESTART_SLEEP_TIME=10
-
-#the minimum and maximum number of tservers the agitator will kill at once
-TSERVER_MIN_KILL=1
-TSERVER_MAX_KILL=1
-
-#the amount of time (in minutes) the agitator should sleep before killing datanodes
-DATANODE_KILL_SLEEP_TIME=20
-
-#the amount of time (in minutes) the agitator should sleep after killing
-# before restarting datanodes
-DATANODE_RESTART_SLEEP_TIME=10
-
-#the minimum and maximum number of datanodes the agitator will kill at once
-DATANODE_MIN_KILL=1
-DATANODE_MAX_KILL=1
-
-#time in minutes between killing masters
-MASTER_KILL_SLEEP_TIME=60
-MASTER_RESTART_SLEEP_TIME=2
-
-#Do we want to perturb HDFS? Only works on HDFS versions with HA, i.e. Hadoop 2
-# AGITATE_HDFS=true
-AGITATE_HDFS=false
-AGITATE_HDFS_SLEEP_TIME=10
-AGITATE_HDFS_SUPERUSER=hdfs
-AGITATE_HDFS_COMMAND="${HADOOP_PREFIX:-/usr/lib/hadoop}/share/hadoop/hdfs/bin/hdfs"
-AGITATE_HDFS_SUDO=$(which sudo)
-
-#settings for the verification map reduce job
-VERIFY_OUT=/tmp/continuous_verify
-VERIFY_MAX_MAPS=64
-VERIFY_REDUCERS=64
-SCAN_OFFLINE=false
-#comma separated list of auths to use for verify
-VERIFY_AUTHS=''
-
-#settings related to the batch walker
-# sleep in seconds
-BATCH_WALKER_SLEEP=1800
-BATCH_WALKER_BATCH_SIZE=10000
-BATCH_WALKER_THREADS=8
-
-#settings related to scanners
-# sleep in seconds
-SCANNER_SLEEP_TIME=10
-SCANNER_ENTRIES=5000
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/datanode-agitator.pl
----------------------------------------------------------------------
diff --git a/continuous/datanode-agitator.pl b/continuous/datanode-agitator.pl
deleted file mode 100755
index a98bb66..0000000
--- a/continuous/datanode-agitator.pl
+++ /dev/null
@@ -1,140 +0,0 @@
-#! /usr/bin/env perl
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-use POSIX qw(strftime);
-use Cwd qw();
-
-if(scalar(@ARGV) != 5 && scalar(@ARGV) != 3){
- print "Usage : datanode-agitator.pl <min sleep before kill in minutes>[:max sleep before kill in minutes] <min sleep before restart in minutes>[:max sleep before restart in minutes] HADOOP_PREFIX [<min kill> <max kill>]\n";
- exit(1);
-}
-
-my $ACCUMULO_HOME;
-if( defined $ENV{'ACCUMULO_HOME'} ){
- $ACCUMULO_HOME = $ENV{'ACCUMULO_HOME'};
-} else {
- $cwd=Cwd::cwd();
- $ACCUMULO_HOME=$cwd . '/../../..';
-}
-$HADOOP_PREFIX=$ARGV[2];
-
-print "ACCUMULO_HOME=$ACCUMULO_HOME\n";
-print "HADOOP_PREFIX=$HADOOP_PREFIX\n";
-
-@sleeprange1 = split(/:/, $ARGV[0]);
-$sleep1 = $sleeprange1[0];
-
-@sleeprange2 = split(/:/, $ARGV[1]);
-$sleep2 = $sleeprange2[0];
-
-if (scalar(@sleeprange1) > 1) {
- $sleep1max = $sleeprange1[1] + 1;
-} else {
- $sleep1max = $sleep1;
-}
-
-if ($sleep1 > $sleep1max) {
- die("sleep1 > sleep1max $sleep1 > $sleep1max");
-}
-
-if (scalar(@sleeprange2) > 1) {
- $sleep2max = $sleeprange2[1] + 1;
-} else {
- $sleep2max = $sleep2;
-}
-
-if($sleep2 > $sleep2max){
- die("sleep2 > sleep2max $sleep2 > $sleep2max");
-}
-
-if(defined $ENV{'ACCUMULO_CONF_DIR'}){
- $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
-}else{
- $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
-}
-
-if(scalar(@ARGV) == 5){
- $minKill = $ARGV[3];
- $maxKill = $ARGV[4];
-}else{
- $minKill = 1;
- $maxKill = 1;
-}
-
-if($minKill > $maxKill){
- die("minKill > maxKill $minKill > $maxKill");
-}
-
-@tserversRaw = `cat $ACCUMULO_CONF_DIR/tservers`;
-chomp(@tserversRaw);
-
-for $tserver (@tserversRaw){
- if($tserver eq "" || substr($tserver,0,1) eq "#"){
- next;
- }
-
- push(@tservers, $tserver);
-}
-
-
-if(scalar(@tservers) < $maxKill){
- print STDERR "WARN setting maxKill to ".scalar(@tservers)."\n";
- $maxKill = scalar(@tservers);
-}
-
-if ($minKill > $maxKill){
- print STDERR "WARN setting minKill to equal maxKill\n";
- $minKill = $maxKill;
-}
-
-while(1){
-
- $numToKill = int(rand($maxKill - $minKill + 1)) + $minKill;
- %killed = ();
- $server = "";
-
- for($i = 0; $i < $numToKill; $i++){
- while($server eq "" || $killed{$server} != undef){
- $index = int(rand(scalar(@tservers)));
- $server = $tservers[$index];
- }
-
- $killed{$server} = 1;
-
- $t = strftime "%Y%m%d %H:%M:%S", localtime;
-
- print STDERR "$t Killing datanode on $server\n";
- system("ssh $server \"pkill -9 -f '[p]roc_datanode'\"");
- }
-
- $nextsleep2 = int(rand($sleep2max - $sleep2)) + $sleep2;
- sleep($nextsleep2 * 60);
-
- foreach $restart (keys %killed) {
-
- $t = strftime "%Y%m%d %H:%M:%S", localtime;
-
- print STDERR "$t Starting datanode on $restart\n";
- # We can just start as we're the HDFS user
- system("ssh $restart '$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode'");
- }
-
- $nextsleep1 = int(rand($sleep1max - $sleep1)) + $sleep1;
- sleep($nextsleep1 * 60);
-}
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/hdfs-agitator.pl
----------------------------------------------------------------------
diff --git a/continuous/hdfs-agitator.pl b/continuous/hdfs-agitator.pl
deleted file mode 100755
index 85eab32..0000000
--- a/continuous/hdfs-agitator.pl
+++ /dev/null
@@ -1,217 +0,0 @@
-#! /usr/bin/env perl
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-use strict;
-use warnings;
-use POSIX qw(strftime);
-use Getopt::Long;
-use Pod::Usage;
-
-my $help = 0;
-my $man = 0;
-my $sleep = 10;
-my $superuser = 'hdfs';
-my $hdfsCmd;
-if( defined $ENV{'HADOOP_PREFIX'} ){
- $hdfsCmd = $ENV{'HADOOP_PREFIX'} . '/share/hadoop/hdfs/bin/hdfs';
-}
-my $sudo;
-my $nameservice;
-
-GetOptions('help|?' => \$help, 'man' => \$man, 'sleep=i' => \$sleep, 'nameservice=s' => \$nameservice, 'superuser=s' => \$superuser, 'hdfs-cmd=s' => \$hdfsCmd, 'sudo:s' => \$sudo) or pod2usage(2);
-pod2usage(-exitval => 0, -verbose => 1) if $help;
-pod2usage(-exitval => 0, -verbose => 2) if $man;
-pod2usage(-exitval => 1, -verbose => 1, -message => '$HADOOP_PREFIX not defined and no hdfs-cmd given. please use --hdfs-cmd to specify where your hdfs cli is.') if not defined $hdfsCmd;
-pod2usage(-exitval => 1, -verbose => 1, -message => "Your specified hdfs cli '$hdfsCmd' is not executable.") if not -x $hdfsCmd;
-if( defined $sudo and "" eq $sudo ){
- $sudo = `which sudo`;
- pod2usage(-exitval => 1, -verbose => 1, -message => "Error attempting to find the sudo command, please specify it with --sudo /path/to/sudo") if 0 != $?;
- chomp($sudo);
-}
-if( defined $sudo ){
- pod2usage(-exitval => 1, -verbose => 1, -message => "Your specified sudo command '$sudo' is not executable.") if not -x $sudo;
-}
-
-my $needsudo = defined $sudo;
-my $haadmin = "$hdfsCmd haadmin";
-if($needsudo) {
- $haadmin = "$sudo -u $superuser $haadmin";
- print STDERR "Starting HDFS agitator, configured to fail over every $sleep minutes. will run hdfs command '$hdfsCmd' as user '$superuser' via '$sudo'.\n";
-} else {
- print STDERR "Starting HDFS agitator, configured to fail over every $sleep minutes. will run hdfs command '$hdfsCmd' as the current user.\n";
-}
-while(1){
- sleep($sleep * 60);
- my $t = strftime "%Y%m%d %H:%M:%S", localtime;
- my @failServices;
- if( defined $nameservice ){
- @failServices = ($nameservice);
- } else {
- my $nameservicesRaw = `$hdfsCmd getconf -confKey dfs.nameservices`;
- if(0 != $?) {
- print STDERR "$t HDFS CLI failed. please see --help to set it correctly\n";
- exit(1);
- }
- chomp($nameservicesRaw);
- my @nameservices = split(/,/, $nameservicesRaw);
- if(1 > scalar(@nameservices)) {
- print STDERR "$t No HDFS NameServices found. Are you sure you're running in HA?\n";
- exit(1);
- }
- if(rand(1) < .5){
- my $serviceToFail = $nameservices[int(rand(scalar(@nameservices)))];
- print STDERR "$t Failing over nameservice $serviceToFail\n";
- @failServices = ($serviceToFail);
- } else {
- print STDERR "$t Failing over all nameservices\n";
- @failServices = @nameservices;
- }
- }
- for my $toFail (@failServices){
- my $namenodesRaw = `$hdfsCmd getconf -confKey dfs.ha.namenodes.$toFail`;
- if(0 != $?) {
- print STDERR "$t HDFS CLI failed to look up namenodes in service $toFail.\n";
- exit(1);
- }
- chomp($namenodesRaw);
- my @namenodes = split(/,/, $namenodesRaw);
- if(2 > scalar(@namenodes)) {
- print STDERR "$t WARN NameService $toFail does not have at least 2 namenodes according to the HDFS configuration, skipping.\n";
- next;
- }
- my $active;
- for my $namenode (@namenodes){
- my $status = `$haadmin -ns $toFail -getServiceState $namenode`;
- if(0 != $?) {
- if($needsudo) {
- print STDERR "$t WARN Error while attempting to get the service state of $toFail :: $namenode\n";
- $status = 'error';
- } else {
- print STDERR "$t WARN Current user may not run the HDFS haadmin utility, attempting to sudo to the $superuser user.\n";
- $needsudo = 1;
- if(not defined $sudo) {
- $sudo = `which sudo`;
- pod2usage(-exitval => 1, -verbose => 1, -message => "Error attempting to find the sudo command, please specify it with --sudo") if 0 != $?;
- chomp($sudo);
- pod2usage(-exitval => 1, -verbose => 1, -message => "The sudo command '$sudo' is not executable. please specify sudo with --sudo") if not -x $sudo;
- }
- $haadmin = "$sudo -u $superuser $haadmin";
- redo;
- }
- }
- chomp($status);
- if( 'active' eq $status ){
- $active = $namenode;
- last;
- }
- }
- if( defined $active ){
- my @standby = grep { $_ ne $active } @namenodes;
- my $newActive = $standby[int(rand(scalar(@standby)))];
- print STDERR "$t Transitioning nameservice $toFail from $active to $newActive\n";
- my $cmd = "$haadmin -ns $toFail -failover $active $newActive";
- print "$t $cmd\n";
- system($cmd);
- } else {
- my $newActive = $namenodes[int(rand(scalar(@namenodes)))];
- print STDERR "$t WARN nameservice $toFail did not have an active namenode. Transitioning a random namenode to active. This will fail if HDFS is configured for automatic failover.\n";
- my $cmd = "$haadmin -ns $toFail -transitionToActive $newActive";
- print "$t $cmd\n";
- system($cmd);
- }
- }
-}
-__END__
-
-=head1 NAME
-
-hdfs-agitator - causes HDFS to failover
-
-=head1 DESCRIPTION
-
-Sleeps for a configurable amount of time, then causes a NameNode failover in one
-or more HDFS NameServices. If a given NameService does not have an Active
-NameNode when it comes time to failover, a random standby is promoted.
-
-Only works on HDFS versions that support HA configurations and the haadmin
-command. In order to function, the user running this script must be able to
-use the haadmin command. This requires access to an HDFS superuser. By default,
-it will attempt to sudo to perform calls.
-
-=head1 SYNOPSIS
-
-hdfs-agitator [options]
-
- Options:
- --help Brief help message
- --man Full documentation
- --sleep Time to sleep between failovers in minutes. Default 10
- --superuser HDFS superuser. Default 'hdfs'
- --hdfs-cmd hdfs command path. Default '$HADOOP_PREFIX/share/hadoop/hdfs/bin/hdfs'
- --nameservice Limit failovers to specified nameservice. Default all nameservices
- --sudo command to call to sudo to the HDFS superuser. Default 'sudo' if needed.
-
-=head1 OPTIONS
-
-=over 8
-
-=item B<--sleep>
-
-Sleep the given number of minutes between attempts to fail over nameservices.
-
-=item B<--nameservice>
-
-Limit failover attempts to the given nameservice. By default, we attempt ot list
-all known nameservices and choose either one or all of them to failover in a
-given cycle.
-
-=item B<--superuser>
-
-An HDFS superuser capable of running the haadmin command. Defaults to "hdfs".
-
-=item B<--hdfs-cmd>
-
-Path to the HDFS cli. Will be used both for non-administrative commands (e.g.
-listing the nameservices and serviceids in a given nameservice) and admin-only
-actions such as checking status and failing over.
-
-Defaults to using $HADOOP_PREFIX.
-
-=item B<--sudo>
-
-Command to allow us to act as the given HDFS superuser. By default we assume the current user
-can run HDFS administrative commands. When this argument is specified we will instead attempt
-to use the HDFS superuser instead. If given an argument, it will be called like
-sudo, i.e. "sudo -u $superuser $cmd". Defaults to "sudo" on the shell's path.
-
-=back
-
-=head1 SEE ALSO
-
-See the Apache Hadoop documentation on configuring HDFS HA
-
-=over 8
-
-=item B<HA with QJM>
-
-http://hadoop.apache.org/docs/r2.2.0/hadoop-yarn/hadoop-yarn-site/HDFSHighAvailabilityWithQJM.html#Administrative_commands
-
-=item B<HA with NFS>
-
-http://hadoop.apache.org/docs/r2.2.0/hadoop-yarn/hadoop-yarn-site/HDFSHighAvailabilityWithNFS.html#Administrative_commands
-
-=back
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/ingesters.txt.example
----------------------------------------------------------------------
diff --git a/continuous/ingesters.txt.example b/continuous/ingesters.txt.example
deleted file mode 100644
index b66d790..0000000
--- a/continuous/ingesters.txt.example
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-host1
-host2
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/master-agitator.pl
----------------------------------------------------------------------
diff --git a/continuous/master-agitator.pl b/continuous/master-agitator.pl
deleted file mode 100755
index d87f17e..0000000
--- a/continuous/master-agitator.pl
+++ /dev/null
@@ -1,92 +0,0 @@
-#! /usr/bin/env perl
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-use POSIX qw(strftime);
-use Cwd qw();
-
-if(scalar(@ARGV) != 2){
- print "Usage : master-agitator.pl <sleep before kill in minutes> <sleep before start in minutes>\n";
- exit(1);
-}
-
-my $ACCUMULO_HOME;
-if( defined $ENV{'ACCUMULO_HOME'} ){
- $ACCUMULO_HOME = $ENV{'ACCUMULO_HOME'};
-} else {
- $cwd=Cwd::cwd();
- $ACCUMULO_HOME=$cwd . '/../../..';
-}
-
-if(defined $ENV{'ACCUMULO_CONF_DIR'}){
- $ACCUMULO_CONF_DIR = $ENV{'ACCUMULO_CONF_DIR'};
-}else{
- $ACCUMULO_CONF_DIR = $ACCUMULO_HOME . '/conf';
-}
-
-$sleep1 = $ARGV[0];
-$sleep2 = $ARGV[1];
-
-@mastersRaw = `cat $ACCUMULO_CONF_DIR/masters`;
-chomp(@mastersRaw);
-
-for $master (@mastersRaw){
- if($master eq "" || substr($master,0,1) eq "#"){
- next;
- }
-
- push(@masters, $master);
-}
-
-
-while(1){
- sleep($sleep1 * 60);
- $t = strftime "%Y%m%d %H:%M:%S", localtime;
- if(rand(1) < .5){
- $masterNodeToWack = $masters[int(rand(scalar(@masters)))];
- print STDERR "$t Killing master on $masterNodeToWack\n";
- $cmd = "ssh $masterNodeToWack \"pkill -f '[ ]org.apache.accumulo.start.*master'\"";
- print "$t $cmd\n";
- system($cmd);
- }else{
- print STDERR "$t Killing all masters\n";
- $cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
- print "$t $cmd\n";
- system($cmd);
-
- $file = '';
- if (-e "$ACCUMULO_CONF_DIR/gc") {
- $file = 'gc';
- } else {
- $file = 'masters';
- }
-
- $cmd = "pssh -h $ACCUMULO_CONF_DIR/$file \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
- print "$t $cmd\n";
- system($cmd);
- }
-
- sleep($sleep2 * 60);
- $t = strftime "%Y%m%d %H:%M:%S", localtime;
- print STDERR "$t Running start-all\n";
-
- $cmd = "pssh -h $ACCUMULO_CONF_DIR/masters \"$ACCUMULO_HOME/bin/accumulo-service master start\" < /dev/null";
- print "$t $cmd\n";
- system($cmd);
-}
-
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/report.pl
----------------------------------------------------------------------
diff --git a/continuous/report.pl b/continuous/report.pl
deleted file mode 100755
index d1902b6..0000000
--- a/continuous/report.pl
+++ /dev/null
@@ -1,120 +0,0 @@
-#! /usr/bin/env perl
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-use POSIX qw(strftime);
-
-if(scalar(@ARGV) == 2 && $ARGV[0] eq "-bt"){
- $BIN_TIME=$ARGV[1];
-}elsif(scalar(@ARGV) == 0){
- $BIN_TIME=900;
-}else{
- print "Usage : report.pl [-bt <bin time>]\n";
- exit;
-}
-
-
-$LOG_DIR = "logs";
-$ACCUMULO_HOME="../../..";
-$REPORT_DIR = strftime "report_%Y%m%d%H%M", localtime;
-
-mkdir("$REPORT_DIR");
-
-open (HTML, ">$REPORT_DIR/report.html");
-
-print HTML "<html><body>\n";
-
-$misCount = `grep MIS $LOG_DIR/*_walk.err | wc -l`;
-
-if($misCount > 0){
- print HTML "<HR width=50% size=4>\n";
- print HTML "<center><P><B color=red>WARNING : The walkers saw missing nodes, this should not happen</B><P></center>\n";
- print HTML "<HR width=50% size=4>\n";
-}
-
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 2, "AVG", "entries", "Entries over time");
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 3, "AMM", "ingest_rate", "Ingest rate over time");
-plot("egrep 'SRQ|FSR' $LOG_DIR/*_walk.out", $BIN_TIME, 1, 3, "AMM", "query_latency", "Row lookup latency (in milliseconds) over time");
-plot("egrep 'SRQ|FSR' $LOG_DIR/*_walk.out", 3600, 1, 3, "COUNT", "query_count", "# rows looked up in each hour");
-plot("grep 'BRQ' $LOG_DIR/*_batch_walk.out", $BIN_TIME, 1, 5, "AMM", "batch_walk_rate", "batch walkers average lookup rate" );
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 10, "AVG", "tablets", "Table tablets online over time");
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 25, "AMM_HACK1", "files_per_tablet", "Files per tablet");
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 1, "AVG", "tservers", "Tablet servers over time");
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 11, "AVG", "du", "HDFS usage over time");
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 12, "AVG", "dirs", "HDFS # dirs over time");
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 13, "AVG", "files", "HDFS # files over time");
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 17, "AVG", "maps", "# map task over time");
-plot("cat $LOG_DIR/*_stats.out", $BIN_TIME, 0, 19, "AVG", "reduces", "# reduce task over time");
-
-print HTML "<P><h2>Config</h2>\n";
-print HTML "<UL>\n";
-for $config_file (glob("$LOG_DIR/*_config.out")){
- @path = split(/\//,$config_file);
- $file_name = $path[$path - 1];
- system("cp $config_file $REPORT_DIR/$file_name");
- print HTML "<li><a href='$file_name'>$file_name</a>\n";
-}
-print HTML "</UL>\n";
-
-
-print HTML "<P><h2>Lookup times histogram</h2>\n";
-print HTML "<pre>\n";
-print HTML `cat $LOG_DIR/*_walk.out | $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.continuous.PrintScanTimeHistogram`;
-print HTML "</pre>\n";
-
-print HTML "</body></html>\n";
-close(HTML);
-
-sub plot {
- my $cmd = shift(@_);
- my $period = shift(@_);
- my $time_col = shift(@_);
- my $data_col = shift(@_);
- my $op = shift(@_);
- my $output = shift(@_);
- my $title = shift(@_);
-
- system("$cmd | $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.continuous.TimeBinner --period $period --timeColumn $time_col --dataColumn $data_col --operation $op --dateFormat MM/dd/yy-HH:mm:ss > $REPORT_DIR/$output.dat");
- gnuplot("$REPORT_DIR/$output.dat", "$REPORT_DIR/$output.png", $op eq "AMM" || $op eq "AMM_HACK1");
-
- print HTML "<P><h2>$title</h2><img src='$output.png'>\n";
-}
-
-sub gnuplot {
- my $input = shift(@_);
- my $output = shift(@_);
- my $yerr = shift(@_);
-
- open(GNUP, "|gnuplot > $output");
-
- print GNUP "set xdata time\n";
- print GNUP "set timefmt \"%m/%d/%y-%H:%M:%S\"\n";
- print GNUP "set format x \"%m/%d\"\n";
- print GNUP "set offsets 1,1,1,1\n";
- print GNUP "set size 1.25,1.25\n";
- print GNUP "set terminal png\n";
- if($yerr){
- print GNUP "plot \"$input\" using 1:2:3:4 with yerrorlines\n";
- }else{
- print GNUP "plot \"$input\" using 1:2\n";
- }
-
- close(GNUP);
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/run-moru.sh
----------------------------------------------------------------------
diff --git a/continuous/run-moru.sh b/continuous/run-moru.sh
deleted file mode 100755
index 3c73ddb..0000000
--- a/continuous/run-moru.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start: Resolve Script Directory
-SOURCE="${BASH_SOURCE[0]}"
-while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink
- bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
- SOURCE="$(readlink "${SOURCE}")"
- [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
-script=$( basename "${SOURCE}" )
-# Stop: Resolve Script Directory
-
-CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-${bin}}
-. $CONTINUOUS_CONF_DIR/continuous-env.sh
-. $ACCUMULO_CONF_DIR/accumulo-env.sh
-
-SERVER_LIBJAR="$ACCUMULO_HOME/lib/accumulo-test.jar"
-
-"$ACCUMULO_HOME/contrib/tool.sh" "$SERVER_LIBJAR" org.apache.accumulo.test.continuous.ContinuousMoru -libjars "$SERVER_LIBJAR" -i "$INSTANCE_NAME" -z "$ZOO_KEEPERS" -u "$USER" -p "$PASS" --table "$TABLE" --min "$MIN" --max "$MAX" --maxColF "$MAX_CF" --maxColQ "$MAX_CQ" --batchMemory "$MAX_MEM" --batchLatency "$MAX_LATENCY" --batchThreads "$NUM_THREADS" --maxMappers "$VERIFY_MAX_MAPS"
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/run-verify.sh
----------------------------------------------------------------------
diff --git a/continuous/run-verify.sh b/continuous/run-verify.sh
deleted file mode 100755
index aa56643..0000000
--- a/continuous/run-verify.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start: Resolve Script Directory
-SOURCE="${BASH_SOURCE[0]}"
-while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink
- bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
- SOURCE=$(readlink "${SOURCE}")
- [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
-script=$( basename "${SOURCE}" )
-# Stop: Resolve Script Directory
-
-CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-${bin}}
-. $CONTINUOUS_CONF_DIR/continuous-env.sh
-. $ACCUMULO_CONF_DIR/accumulo-env.sh
-
-SERVER_LIBJAR="$ACCUMULO_HOME/lib/accumulo-test.jar"
-
-AUTH_OPT="";
-[[ -n $VERIFY_AUTHS ]] && AUTH_OPT="--auths $VERIFY_AUTHS"
-
-SCAN_OPT=--offline
-[[ $SCAN_OFFLINE == false ]] && SCAN_OPT=
-
-"$ACCUMULO_HOME/contrib/tool.sh" "$SERVER_LIBJAR" org.apache.accumulo.test.continuous.ContinuousVerify -Dmapreduce.job.reduce.slowstart.completedmaps=0.95 -libjars "$SERVER_LIBJAR" "$AUTH_OPT" -i "$INSTANCE_NAME" -z "$ZOO_KEEPERS" -u "$USER" -p "$PASS" --table "$TABLE" --output "$VERIFY_OUT" --maxMappers "$VERIFY_MAX_MAPS" --reducers "$VERIFY_REDUCERS" "$SCAN_OPT"
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/scanners.txt.example
----------------------------------------------------------------------
diff --git a/continuous/scanners.txt.example b/continuous/scanners.txt.example
deleted file mode 100644
index 63fb8bb..0000000
--- a/continuous/scanners.txt.example
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-localhost
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/start-agitator.sh
----------------------------------------------------------------------
diff --git a/continuous/start-agitator.sh b/continuous/start-agitator.sh
deleted file mode 100755
index a44cd83..0000000
--- a/continuous/start-agitator.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Start: Resolve Script Directory
-SOURCE="${BASH_SOURCE[0]}"
-while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink
- bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
- SOURCE=$(readlink "${SOURCE}")
- [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
-script=$( basename "${SOURCE}" )
-# Stop: Resolve Script Directory
-
-CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-${bin}}
-. "$CONTINUOUS_CONF_DIR/continuous-env.sh"
-
-mkdir -p "$CONTINUOUS_LOG_DIR"
-
-LOG_BASE="${CONTINUOUS_LOG_DIR}/$(date +%Y%m%d%H%M%S)_$(hostname)"
-
-# Start agitators for datanodes, tservers, and the master
-[[ -n $AGITATOR_USER ]] || AGITATOR_USER=$(whoami)
-if [[ $AGITATOR_USER == root ]]; then
- echo "Running master-agitator and tserver-agitator as $ACCUMULO_USER using su. Running datanode-agitator as $HDFS_USER using su."
-
- # Change to the correct user if started as root
- su -c "nohup ${bin}/master-agitator.pl $MASTER_KILL_SLEEP_TIME $MASTER_RESTART_SLEEP_TIME >${LOG_BASE}_master-agitator.out 2>${LOG_BASE}_master-agitator.err &" -m - "$ACCUMULO_USER"
-
- su -c "nohup ${bin}/tserver-agitator.pl $TSERVER_KILL_SLEEP_TIME $TSERVER_RESTART_SLEEP_TIME $TSERVER_MIN_KILL $TSERVER_MAX_KILL >${LOG_BASE}_tserver-agitator.out 2>${LOG_BASE}_tserver-agitator.err &" -m - "$ACCUMULO_USER"
-
- su -c "nohup ${bin}/datanode-agitator.pl $DATANODE_KILL_SLEEP_TIME $DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $DATANODE_MIN_KILL $DATANODE_MAX_KILL >${LOG_BASE}_datanode-agitator.out 2>${LOG_BASE}_datanode-agitator.err &" -m - "$HDFS_USER"
-
-elif [[ $AGITATOR_USER == "$ACCUMULO_USER" ]]; then
- echo "Running master-agitator and tserver-agitator as $AGITATOR_USER Running datanode-agitator as $HDFS_USER using sudo."
- # Just run the master-agitator if we're the accumulo user
- nohup "${bin}/master-agitator.pl" "$MASTER_KILL_SLEEP_TIME" "$MASTER_RESTART_SLEEP_TIME" >"${LOG_BASE}_master-agitator.out" 2>"${LOG_BASE}_master-agitator.err" &
-
- nohup "${bin}/tserver-agitator.pl" "$TSERVER_KILL_SLEEP_TIME" "$TSERVER_RESTART_SLEEP_TIME" "$TSERVER_MIN_KILL" "$TSERVER_MAX_KILL" >"${LOG_BASE}_tserver-agitator.out" 2>"${LOG_BASE}_tserver-agitator.err" &
-
- sudo -u "$HDFS_USER" nohup "${bin}/datanode-agitator.pl" "$DATANODE_KILL_SLEEP_TIME" "$DATANODE_RESTART_SLEEP_TIME" "$HADOOP_PREFIX" "$DATANODE_MIN_KILL" "$DATANODE_MAX_KILL" >"${LOG_BASE}_datanode-agitator.out" 2>"${LOG_BASE}_datanode-agitator.err" &
-
-else
- echo "Running master-agitator and tserver-agitator as $ACCUMULO_USER using sudo. Running datanode-agitator as $HDFS_USER using sudo."
-
- # Not root, and not the accumulo user, hope you can sudo to it
- sudo -u "$ACCUMULO_USER" "nohup ${bin}/master-agitator.pl $MASTER_KILL_SLEEP_TIME $MASTER_RESTART_SLEEP_TIME >${LOG_BASE}_master-agitator.out 2>${LOG_BASE}_master-agitator.err &"
-
- sudo -u "$ACCUMULO_USER" "nohup ${bin}/tserver-agitator.pl $TSERVER_KILL_SLEEP_TIME $TSERVER_RESTART_SLEEP_TIME $TSERVER_MIN_KILL $TSERVER_MAX_KILL >${LOG_BASE}_tserver-agitator.out 2>${LOG_BASE}_tserver-agitator.err &"
-
- sudo -u "$HDFS_USER" "nohup ${bin}/datanode-agitator.pl $DATANODE_KILL_SLEEP_TIME $DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $DATANODE_MIN_KILL $DATANODE_MAX_KILL >${LOG_BASE}_datanode-agitator.out 2>${LOG_BASE}_datanode-agitator.err &" -m - "$HDFS_USER"
-
-fi
-
-if ${AGITATE_HDFS:-false} ; then
- AGITATOR_LOG=${LOG_BASE}_hdfs-agitator
- sudo -u "$AGITATE_HDFS_SUPERUSER" nohup "${bin}/hdfs-agitator.pl" --sleep "${AGITATE_HDFS_SLEEP_TIME}" --hdfs-cmd "${AGITATE_HDFS_COMMAND}" --superuser "${AGITATE_HDFS_SUPERUSER}" >"${AGITATOR_LOG}.out" 2>"${AGITATOR_LOG}.err" &
-fi
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/start-batchwalkers.sh
----------------------------------------------------------------------
diff --git a/continuous/start-batchwalkers.sh b/continuous/start-batchwalkers.sh
deleted file mode 100755
index 7d4efff..0000000
--- a/continuous/start-batchwalkers.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start: Resolve Script Directory
-SOURCE="${BASH_SOURCE[0]}"
-while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink
- bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
- SOURCE=$(readlink "${SOURCE}")
- [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
-script=$( basename "${SOURCE}" )
-# Stop: Resolve Script Directory
-
-CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-${bin}}
-. "$CONTINUOUS_CONF_DIR/continuous-env.sh"
-
-DEBUG_OPT=''
-if [[ $DEBUG_BATCH_WALKER == on ]] ; then
- DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.log";
-fi
-
-AUTH_OPT=''
-[[ -n $AUTHS ]] && AUTH_OPT="--auths \"$AUTHS\""
-
-pssh -h "$CONTINUOUS_CONF_DIR/batch_walkers.txt" "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.continuous.ContinuousBatchWalker $DEBUG_OPT $AUTH_OPT -i $INSTANCE_NAME -z $ZOO_KEEPERS -u $USER -p $PASS --table $TABLE --min $MIN --max $MAX --sleep $BATCH_WALKER_SLEEP --numToScan $BATCH_WALKER_BATCH_SIZE --scanThreads $BATCH_WALKER_THREADS >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_batch_walk.err &" < /dev/null
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/start-ingest.sh
----------------------------------------------------------------------
diff --git a/continuous/start-ingest.sh b/continuous/start-ingest.sh
deleted file mode 100755
index 8cc7d07..0000000
--- a/continuous/start-ingest.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start: Resolve Script Directory
-SOURCE="${BASH_SOURCE[0]}"
-while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink
- bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
- SOURCE=$(readlink "${SOURCE}")
- [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
-script=$( basename "${SOURCE}" )
-# Stop: Resolve Script Directory
-
-CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-${bin}}
-. "$CONTINUOUS_CONF_DIR/continuous-env.sh"
-
-DEBUG_OPT=''
-if [[ $DEBUG_INGEST == on ]] ; then
- DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.log";
-fi
-
-VIS_OPT=''
-[[ -n $VISIBILITIES ]] && VIS_OPT="--visibilities \"$VISIBILITIES\""
-
-CHECKSUM_OPT='--addCheckSum'
-[[ $CHECKSUM == false ]] && CHECKSUM_OPT=''
-
-pssh -h "$CONTINUOUS_CONF_DIR/ingesters.txt" "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.continuous.ContinuousIngest $DEBUG_OPT $VIS_OPT -i $INSTANCE_NAME -z $ZOO_KEEPERS -u $USER -p $PASS --table $TABLE --num $NUM --min $MIN --max $MAX --maxColF $MAX_CF --maxColQ $MAX_CQ --batchMemory $MAX_MEM --batchLatency $MAX_LATENCY --batchThreads $NUM_THREADS $CHECKSUM_OPT >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_ingest.err &" < /dev/null
-
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/fc3ddfc4/continuous/start-scanners.sh
----------------------------------------------------------------------
diff --git a/continuous/start-scanners.sh b/continuous/start-scanners.sh
deleted file mode 100755
index c876768..0000000
--- a/continuous/start-scanners.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Start: Resolve Script Directory
-SOURCE="${BASH_SOURCE[0]}"
-while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink
- bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
- SOURCE=$(readlink "${SOURCE}")
- [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-bin=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
-script=$( basename "${SOURCE}" )
-# Stop: Resolve Script Directory
-
-CONTINUOUS_CONF_DIR=${CONTINUOUS_CONF_DIR:-${bin}}
-. "$CONTINUOUS_CONF_DIR/continuous-env.sh"
-
-DEBUG_OPT="";
-if [[ "$DEBUG_SCANNER" == "on" ]] ; then
- DEBUG_OPT="--debug $CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.log";
-fi
-
-AUTH_OPT="";
-[[ -n "$AUTHS" ]] && AUTH_OPT="--auths \"$AUTHS\""
-
-pssh -h "$CONTINUOUS_CONF_DIR/scanners.txt" "mkdir -p $CONTINUOUS_LOG_DIR; nohup $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.continuous.ContinuousScanner $DEBUG_OPT $AUTH_OPT -i $INSTANCE_NAME -z $ZOO_KEEPERS -u $USER -p $PASS --table $TABLE --min $MIN --max $MAX --sleep $SCANNER_SLEEP_TIME --numToScan $SCANNER_ENTRIES >$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.out 2>$CONTINUOUS_LOG_DIR/\`date +%Y%m%d%H%M%S\`_\`hostname\`_scanner.err &" < /dev/null
-