You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/01/25 18:21:51 UTC
[2/5] accumulo-testing git commit: ACCUMULO-4510 Moved remaining
external test code from Accumulo
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/diffAPI.pl
----------------------------------------------------------------------
diff --git a/test/compat/diffAPI.pl b/test/compat/diffAPI.pl
new file mode 100755
index 0000000..183655d
--- /dev/null
+++ b/test/compat/diffAPI.pl
@@ -0,0 +1,104 @@
+#!/usr/bin/perl
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sub trim($)
+{
+ my $string = shift;
+ $string =~ s/^\s+//;
+ $string =~ s/\s+$//;
+ return $string;
+}
+
+sub getDeprecated {
+ my($jar, $class) = @_;
+
+ open(JAVAP, "javap -verbose -public -classpath '$jar' '$class'|");
+
+ my $lastMethod = "";
+ my %deprecated;
+
+ while(<JAVAP>){
+ chomp();
+ if(/^public\s/){
+ $lastMethod = $_;
+ }
+ if(/Deprecated\:\strue/){
+ $lastMethod =~ s/\s+/ /g;
+ $deprecated{$lastMethod}="true";
+ }
+ }
+
+ close(JAVAP);
+
+ return %deprecated;
+}
+
+sub annotateDeprecated {
+ my($jar, $class, $deprecated, $outFile) = @_;
+ open(JAVAP, "javap -public -classpath '$jar' '$class'|");
+ open(OUT, ">$outFile");
+ my @javapOut = <JAVAP>;
+ @javapOut = sort(@javapOut);
+
+ for my $line (@javapOut){
+ my $trimLine = trim($line);
+ chomp($line);
+ $trimLine =~ s/\s+/ /g;
+ if($deprecated->{$trimLine}){
+ print OUT "$line DEPRECATED\n";
+ }else{
+ print OUT "$line\n";
+ }
+ }
+
+ close(JAVAP);
+ close(OUT);
+
+}
+
+if(scalar(@ARGV) != 2){
+ print "Usage : diffAPI.pl <core jar 1> <core jar 2>\n";
+ exit(-1);
+}
+
+$jar1 = $ARGV[0];
+$jar2 = $ARGV[1];
+
+$gtCmd = 'egrep "accumulo/core/client/.*class|accumulo/core/data/.*class" | grep -v accumulo/core/client/impl | grep -v accumulo/core/data/thrift | egrep -v "Impl.*class$" | tr / .';
+
+@classes1 = `jar tf $jar1 | $gtCmd`;
+@classes2 = `jar tf $jar2 | $gtCmd`;
+
+mkdir("diffWorkDir");
+mkdir("diffWorkDir/jar1");
+mkdir("diffWorkDir/jar2");
+
+for $class (@classes1){
+ $class = substr($class, 0, length($class) - 7);
+ %deprecated = getDeprecated($jar1, $class);
+ annotateDeprecated($jar1, $class, \%deprecated, "diffWorkDir/jar1/$class");
+}
+
+for $class (@classes2){
+ $class = substr($class, 0, length($class) - 7);
+ %deprecated = getDeprecated($jar2, $class);
+ annotateDeprecated($jar2, $class, \%deprecated, "diffWorkDir/jar2/$class");
+}
+
+system("diff -u diffWorkDir/jar1 diffWorkDir/jar2");
+system("rm -rf diffWorkDir");
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/README
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/README b/test/compat/japi-compliance/README
new file mode 100644
index 0000000..0b77050
--- /dev/null
+++ b/test/compat/japi-compliance/README
@@ -0,0 +1,53 @@
+Title: Java API Compliance Checker Instructions
+Notice: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ .
+ http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+There is a tool that can analyze the difference between APIs called
+japi-compliance[1]. This tool is useful for checking API compatability of
+different Accumulo versions. To run this tool edit the xml files to specify
+the location of accumulo core jars and set the library version. Then run the
+following command.
+
+ japi-compliance-checker.pl -skip-deprecated -old japi-accumulo-1.5.xml -new japi-accumulo-1.6.xml -l accumulo
+
+Optionally, you can use the --skip-classes argument with the provided exclude_classes.txt file to skip classes from
+org.apache.accumulo.core.data that aren't in the public API.
+
+This directory should have a library configuration file for each release on supported lines as well as an in-progress
+for whatever version is currently the master branch. The examples below all make use of version-specific library definitions.
+
+When looking at a patch release, you should verify that changes introduced are forwards and backwards compatible, per
+semver.
+
+ # Backwards compatibility from x.y.z to x.y.(z+1)
+ japi-compliance-checker.pl -old japi-accumulo-1.6.1.xml -new japi-accumulo-1.6.2.xml -l accumulo --skip-classes=exclude_classes.txt
+ # Forwards compatibility from x.y.z to x.y.(z+1). Note that the old / new arguments have been swapped.
+ japi-compliance-checker.pl -new japi-accumulo-1.6.1.xml -old japi-accumulo-1.6.2.xml -l accumulo --skip-classes=exclude_classes.txt
+
+When looking at a minor release, you should verify that change are backwards compatible, per semver.
+
+ # Backwards compatibility from x.y.z to x.(y+1).0
+ japi-compliance-checker.pl -old japi-accumulo-1.6.1.xml -new japi-accumulo-1.7.0.xml -l accumulo --skip-classes=exclude_classes.txt
+
+When looking at a major release, you should examine removals to make sure they are not capricious. Specifically, you should ensure that
+they have been deprecated for a full major version.
+
+ # Advisory backwards compatibility check from x.y.z to (x+1).0.0
+ japi-compliance-checker.pl -old japi-accumulo-1.7.0.xml -new japi-accumulo-2.0.0.xml -l accumulo --skip-classes=exclude_classes.txt
+
+[1]: http://ispras.linuxbase.org/index.php/Java_API_Compliance_Checker
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/exclude_classes.txt
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/exclude_classes.txt b/test/compat/japi-compliance/exclude_classes.txt
new file mode 100644
index 0000000..94356b7
--- /dev/null
+++ b/test/compat/japi-compliance/exclude_classes.txt
@@ -0,0 +1 @@
+org.apache.accumulo.core.data.KeyValue
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/japi-accumulo-1.5.0.xml
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/japi-accumulo-1.5.0.xml b/test/compat/japi-compliance/japi-accumulo-1.5.0.xml
new file mode 100644
index 0000000..f49dbb5
--- /dev/null
+++ b/test/compat/japi-compliance/japi-accumulo-1.5.0.xml
@@ -0,0 +1,36 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<version>
+ 1.5.0
+</version>
+
+<archives>
+ CHANGEME/accumulo-1.5.0/lib/accumulo-core.jar
+ CHANGEME/accumulo-1.5.0/lib/accumulo-minicluster.jar
+</archives>
+
+<skip_packages>
+ org.apache.accumulo.core.client.impl
+ org.apache.accumulo.core.data.thrift
+</skip_packages>
+
+<packages>
+ org.apache.accumulo.core.client
+ org.apache.accumulo.core.data
+ org.apache.accumulo.minicluster
+</packages>
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/japi-accumulo-1.5.1.xml
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/japi-accumulo-1.5.1.xml b/test/compat/japi-compliance/japi-accumulo-1.5.1.xml
new file mode 100644
index 0000000..ff92506
--- /dev/null
+++ b/test/compat/japi-compliance/japi-accumulo-1.5.1.xml
@@ -0,0 +1,36 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<version>
+ 1.5.1
+</version>
+
+<archives>
+ CHANGEME/accumulo-1.5.1/lib/accumulo-core.jar
+ CHANGEME/accumulo-1.5.1/lib/accumulo-minicluster.jar
+</archives>
+
+<skip_packages>
+ org.apache.accumulo.core.client.impl
+ org.apache.accumulo.core.data.thrift
+</skip_packages>
+
+<packages>
+ org.apache.accumulo.core.client
+ org.apache.accumulo.core.data
+ org.apache.accumulo.minicluster
+</packages>
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/japi-accumulo-1.5.2.xml
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/japi-accumulo-1.5.2.xml b/test/compat/japi-compliance/japi-accumulo-1.5.2.xml
new file mode 100644
index 0000000..8d7a668
--- /dev/null
+++ b/test/compat/japi-compliance/japi-accumulo-1.5.2.xml
@@ -0,0 +1,36 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<version>
+ 1.5.2
+</version>
+
+<archives>
+ CHANGEME/accumulo-1.5.2/lib/accumulo-core.jar
+ CHANGEME/accumulo-1.5.2/lib/accumulo-minicluster.jar
+</archives>
+
+<skip_packages>
+ org.apache.accumulo.core.client.impl
+ org.apache.accumulo.core.data.thrift
+</skip_packages>
+
+<packages>
+ org.apache.accumulo.core.client
+ org.apache.accumulo.core.data
+ org.apache.accumulo.minicluster
+</packages>
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/japi-accumulo-1.6.0.xml
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/japi-accumulo-1.6.0.xml b/test/compat/japi-compliance/japi-accumulo-1.6.0.xml
new file mode 100644
index 0000000..cb1c22f
--- /dev/null
+++ b/test/compat/japi-compliance/japi-accumulo-1.6.0.xml
@@ -0,0 +1,38 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<version>
+ 1.6.0
+</version>
+
+<archives>
+ CHANGEME/accumulo-1.6.0/core/target/accumulo-core-1.6.0.jar
+ CHANGEME/accumulo-1.6.0/minicluster/target/accumulo-minicluster-1.6.0.jar
+</archives>
+
+<skip_packages>
+ org.apache.accumulo.core.client.impl
+ org.apache.accumulo.core.client.lexicoders.impl
+ org.apache.accumulo.core.client.mapreduce.lib.impl
+ org.apache.accumulo.core.data.thrift
+ org.apache.accumulo.minicluster.impl
+</skip_packages>
+
+<packages>
+ org.apache.accumulo.core.client
+ org.apache.accumulo.core.data
+ org.apache.accumulo.minicluster
+</packages>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/japi-accumulo-1.6.1.xml
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/japi-accumulo-1.6.1.xml b/test/compat/japi-compliance/japi-accumulo-1.6.1.xml
new file mode 100644
index 0000000..30e6068
--- /dev/null
+++ b/test/compat/japi-compliance/japi-accumulo-1.6.1.xml
@@ -0,0 +1,38 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<version>
+ 1.6.1
+</version>
+
+<archives>
+ CHANGEME/accumulo-1.6.1/core/target/accumulo-core-1.6.1.jar
+ CHANGEME/accumulo-1.6.1/minicluster/target/accumulo-minicluster-1.6.1.jar
+</archives>
+
+<skip_packages>
+ org.apache.accumulo.core.client.impl
+ org.apache.accumulo.core.client.lexicoders.impl
+ org.apache.accumulo.core.client.mapreduce.lib.impl
+ org.apache.accumulo.core.data.thrift
+ org.apache.accumulo.minicluster.impl
+</skip_packages>
+
+<packages>
+ org.apache.accumulo.core.client
+ org.apache.accumulo.core.data
+ org.apache.accumulo.minicluster
+</packages>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/japi-accumulo-1.6.2.xml
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/japi-accumulo-1.6.2.xml b/test/compat/japi-compliance/japi-accumulo-1.6.2.xml
new file mode 100644
index 0000000..8327206
--- /dev/null
+++ b/test/compat/japi-compliance/japi-accumulo-1.6.2.xml
@@ -0,0 +1,38 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<version>
+ 1.6.2
+</version>
+
+<archives>
+ CHANGEME/accumulo-1.6.2/core/target/accumulo-core-1.6.2.jar
+ CHANGEME/accumulo-1.6.2/minicluster/target/accumulo-minicluster-1.6.2.jar
+</archives>
+
+<skip_packages>
+ org.apache.accumulo.core.client.impl
+ org.apache.accumulo.core.client.lexicoders.impl
+ org.apache.accumulo.core.client.mapreduce.lib.impl
+ org.apache.accumulo.core.data.thrift
+ org.apache.accumulo.minicluster.impl
+</skip_packages>
+
+<packages>
+ org.apache.accumulo.core.client
+ org.apache.accumulo.core.data
+ org.apache.accumulo.minicluster
+</packages>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/japi-accumulo-1.7.0.xml
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/japi-accumulo-1.7.0.xml b/test/compat/japi-compliance/japi-accumulo-1.7.0.xml
new file mode 100644
index 0000000..6caee4d
--- /dev/null
+++ b/test/compat/japi-compliance/japi-accumulo-1.7.0.xml
@@ -0,0 +1,38 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<version>
+ 1.7.0
+</version>
+
+<archives>
+ ../../../core/target/accumulo-core-1.7.0.jar
+ ../../../minicluster/target/accumulo-minicluster-1.7.0.jar
+</archives>
+
+<skip_packages>
+ org.apache.accumulo.core.client.impl
+ org.apache.accumulo.core.client.lexicoders.impl
+ org.apache.accumulo.core.client.mapreduce.lib.impl
+ org.apache.accumulo.core.data.thrift
+ org.apache.accumulo.minicluster.impl
+</skip_packages>
+
+<packages>
+ org.apache.accumulo.core.client
+ org.apache.accumulo.core.data
+ org.apache.accumulo.minicluster
+</packages>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/compat/japi-compliance/japi-accumulo-master.xml
----------------------------------------------------------------------
diff --git a/test/compat/japi-compliance/japi-accumulo-master.xml b/test/compat/japi-compliance/japi-accumulo-master.xml
new file mode 100644
index 0000000..2fc184e
--- /dev/null
+++ b/test/compat/japi-compliance/japi-accumulo-master.xml
@@ -0,0 +1,38 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<version>
+ 1.7.0-SNAPSHOT
+</version>
+
+<archives>
+ ../../../core/target/accumulo-core-1.7.0-SNAPSHOT.jar
+ ../../../minicluster/target/accumulo-minicluster-1.7.0-SNAPSHOT.jar
+</archives>
+
+<skip_packages>
+ org.apache.accumulo.core.client.impl
+ org.apache.accumulo.core.client.lexicoders.impl
+ org.apache.accumulo.core.client.mapreduce.lib.impl
+ org.apache.accumulo.core.data.thrift
+ org.apache.accumulo.minicluster.impl
+</skip_packages>
+
+<packages>
+ org.apache.accumulo.core.client
+ org.apache.accumulo.core.data
+ org.apache.accumulo.minicluster
+</packages>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/merkle-replication/README
----------------------------------------------------------------------
diff --git a/test/merkle-replication/README b/test/merkle-replication/README
new file mode 100644
index 0000000..b892491
--- /dev/null
+++ b/test/merkle-replication/README
@@ -0,0 +1,65 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Distributed Replication Test
+===========================
+
+Ingests random data into a table configured for replication, and then
+verifies that the original table and the replicated table have equivalent
+data using a Merkle tree.
+
+* Steps to run test
+
+1. Configure merkle-env.sh
+
+ Be sure to set the username/password to connect to Accumulo and
+ the Accumulo instance name and ZooKeepers. The example defaults
+ to using the same instance as the source and destination.
+
+ The randomness of the generated data can be controlled via some
+ parameters in this file. Note that if deletes are introduced, it
+ is very likely to cause an incorrect verification since the tombstone'ing
+ of that delete will differ on the source and remote due to the order
+ of minor compactions that occur.
+
+2. Run configure-replication.sh
+
+ This script sets up the two instances for replication, creates
+ the tables that will be replicated and adds some splits to them.
+
+ This is destructive to any existing replication configuration so
+ use it with care on a real instance.
+
+3. Run ingest-data.sh
+
+ Ingests the configured amount of random data into the source
+ table.
+
+4. Run 'accumulo-cluster stop' && 'accumulo-cluster start' on the source instance
+
+ A tabletserver in the source instance is likely to still be referencing
+ a WAL for a presently online tablet which will prevent that
+ file from being replicated. Stopping the local instance will ensure
+ that all WALs are candidate for replication.
+
+5. Run verify-data.sh
+
+ This will compute the leaves merkle tree for the source and destination
+ tables and then compute the root hash for both. The root hash
+ is presented to the user.
+
+ If the root hashes are equal, the test passed; otherwise, the test fails.
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/merkle-replication/configure-replication.sh
----------------------------------------------------------------------
diff --git a/test/merkle-replication/configure-replication.sh b/test/merkle-replication/configure-replication.sh
new file mode 100755
index 0000000..44ebdd7
--- /dev/null
+++ b/test/merkle-replication/configure-replication.sh
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start: Resolve Script Directory
+SOURCE="${BASH_SOURCE[0]}"
+while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink
+ dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+ SOURCE=$(readlink "${SOURCE}")
+ [[ "${SOURCE}" != /* ]] && SOURCE="${dir}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+script=$( basename "${SOURCE}" )
+# Stop: Resolve Script Directory
+
+# Guess at ACCUMULO_HOME and ACCUMULO_CONF_DIR if not already defined
+ACCUMULO_HOME=${ACCUMULO_HOME:-"${dir}/../../.."}
+ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-"$ACCUMULO_HOME/conf"}
+
+# Get the configuration values
+. ./merkle-env.sh
+
+tmpdir=$(mktemp -dt "$0.XXXXXXXXXX")
+
+source_commands="${tmpdir}/source_commands.txt"
+
+echo 'Removing old tables and setting replication name on source'
+
+echo "deletetable -f $SOURCE_TABLE_NAME" >> $source_commands
+echo "createtable $SOURCE_TABLE_NAME" >> $source_commands
+echo "config -s replication.name=source" >> $source_commands
+echo "quit" >> $source_commands
+
+# Source: drop and create tables, configure unique name for replication and grant perms
+echo $SOURCE_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $SOURCE_ACCUMULO_USER -z \
+ $SOURCE_INSTANCE $SOURCE_ZOOKEEPERS -f $source_commands
+
+destination_commands="${tmpdir}/destination_commands.txt"
+
+echo 'Removing old tables and setting replication name on destination'
+
+echo "deletetable -f $DESTINATION_TABLE_NAME" >> $destination_commands
+echo "createtable $DESTINATION_TABLE_NAME" >> $destination_commands
+echo "config -s replication.name=destination" >> $destination_commands
+echo "quit" >> $destination_commands
+
+# Destination: drop and create tables, configure unique name for replication and grant perms
+echo $DESTINATION_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $DESTINATION_ACCUMULO_USER -z \
+ $DESTINATION_INSTANCE $DESTINATION_ZOOKEEPERS -f $destination_commands
+
+rm $source_commands
+rm $destination_commands
+
+table_id=$(echo $DESTINATION_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $DESTINATION_ACCUMULO_USER -z \
+ $DESTINATION_INSTANCE $DESTINATION_ZOOKEEPERS -e 'tables -l' | grep "${DESTINATION_TABLE_NAME}" \
+ | grep -v "${DESTINATION_MERKLE_TABLE_NAME}" | awk '{print $3}')
+
+echo "Configuring $SOURCE_TABLE_NAME to replicate to $DESTINATION_TABLE_NAME (id=$table_id)"
+
+# Define our peer 'destination' with the ReplicaSystem impl, instance name and ZKs
+echo "config -s replication.peer.destination=org.apache.accumulo.tserver.replication.AccumuloReplicaSystem,$DESTINATION_INSTANCE,$DESTINATION_ZOOKEEPERS" >> $source_commands
+# Username for 'destination'
+echo "config -s replication.peer.user.destination=$DESTINATION_ACCUMULO_USER" >> $source_commands
+# Password for 'destination'
+echo "config -s replication.peer.password.destination=$DESTINATION_ACCUMULO_PASSWORD" >> $source_commands
+# Configure replication to 'destination' for $SOURCE_TABLE_NAME
+echo "config -t $SOURCE_TABLE_NAME -s table.replication.target.destination=$table_id" >> $source_commands
+# Enable replication for the table
+echo "config -t $SOURCE_TABLE_NAME -s table.replication=true" >> $source_commands
+echo "quit" >> $source_commands
+
+# Configure replication from source to destination and then enable it
+echo $SOURCE_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $SOURCE_ACCUMULO_USER -z \
+ $SOURCE_INSTANCE $SOURCE_ZOOKEEPERS -f $source_commands
+
+rm $source_commands
+
+# Add some splits to make ingest faster
+echo 'Adding splits...'
+
+echo $SOURCE_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $SOURCE_ACCUMULO_USER -z \
+ $SOURCE_INSTANCE $SOURCE_ZOOKEEPERS -e "addsplits -t $SOURCE_TABLE_NAME 1 2 3 4 5 6 7 8 9"
+
+echo $DESTINATION_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $DESTINATION_ACCUMULO_USER -z \
+ $DESTINATION_INSTANCE $DESTINATION_ZOOKEEPERS -e "addsplits -t $DESTINATION_TABLE_NAME 1 2 3 4 5 6 7 8 9"
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/merkle-replication/ingest-data.sh
----------------------------------------------------------------------
diff --git a/test/merkle-replication/ingest-data.sh b/test/merkle-replication/ingest-data.sh
new file mode 100755
index 0000000..91b8ccc
--- /dev/null
+++ b/test/merkle-replication/ingest-data.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start: Resolve Script Directory
+SOURCE="${BASH_SOURCE[0]}"
+while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink
+ dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+ SOURCE=$(readlink "${SOURCE}")
+ [[ "${SOURCE}" != /* ]] && SOURCE="${dir}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+script=$( basename "${SOURCE}" )
+# Stop: Resolve Script Directory
+
+# Guess at ACCUMULO_HOME and ACCUMULO_CONF_DIR if not already defined
+ACCUMULO_HOME=${ACCUMULO_HOME:-"${dir}/../../.."}
+ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-"$ACCUMULO_HOME/conf"}
+
+# Get the configuration values
+. ./merkle-env.sh
+
+# Ingest data into the source table
+$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.ingest.RandomWorkload --table $SOURCE_TABLE_NAME \
+ -i $SOURCE_INSTANCE -z $SOURCE_ZOOKEEPERS -u $SOURCE_ACCUMULO_USER -p $SOURCE_ACCUMULO_PASSWORD -d $DELETE_PERCENT \
+ -cf $MAX_CF -cq $MAX_CQ -r $MAX_ROW -n $NUM_RECORDS
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/merkle-replication/merkle-env.sh
----------------------------------------------------------------------
diff --git a/test/merkle-replication/merkle-env.sh b/test/merkle-replication/merkle-env.sh
new file mode 100755
index 0000000..d405394
--- /dev/null
+++ b/test/merkle-replication/merkle-env.sh
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Random data will be written to this table
+SOURCE_TABLE_NAME='replicationSource'
+# Then replicated to this table
+DESTINATION_TABLE_NAME='replicationDestination'
+
+# Merkle tree to be stored in this table for the source table
+SOURCE_MERKLE_TABLE_NAME="${SOURCE_TABLE_NAME}_merkle"
+# Merkle tree to be stored in this table for the destination table
+DESTINATION_MERKLE_TABLE_NAME="${DESTINATION_TABLE_NAME}_merkle"
+
+# Connection information to Accumulo
+SOURCE_ACCUMULO_USER="user"
+SOURCE_ACCUMULO_PASSWORD="password"
+
+DESTINATION_ACCUMULO_USER="${SOURCE_ACCUMULO_USER}"
+DESTINATION_ACCUMULO_PASSWORD="${SOURCE_ACCUMULO_PASSWORD}"
+
+SOURCE_INSTANCE="accumulo"
+DESTINATION_INSTANCE="${SOURCE_INSTANCE}"
+
+SOURCE_ZOOKEEPERS="localhost"
+DESTINATION_ZOOKEEPERS="${SOURCE_ZOOKEEPERS}"
+
+# Accumulo user to be configured on the destination instance
+#REPLICATION_USER="${ACCUMULO_USER}"
+#REPLICATION_PASSWORD="${ACCUMULO_PASSWORD}"
+
+# Control amount and distribution of data written
+NUM_RECORDS=100000000
+MAX_ROW=1000000
+MAX_CF=10
+MAX_CQ=100
+DELETE_PERCENT=0
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/merkle-replication/verify-data.sh
----------------------------------------------------------------------
diff --git a/test/merkle-replication/verify-data.sh b/test/merkle-replication/verify-data.sh
new file mode 100755
index 0000000..225d892
--- /dev/null
+++ b/test/merkle-replication/verify-data.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start: Resolve Script Directory
+SOURCE="${BASH_SOURCE[0]}"
+while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink
+ dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+ SOURCE=$(readlink "${SOURCE}")
+ [[ "${SOURCE}" != /* ]] && SOURCE="${dir}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+dir=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+script=$( basename "${SOURCE}" )
+# Stop: Resolve Script Directory
+
+# Guess at ACCUMULO_HOME and ACCUMULO_CONF_DIR if not already defined
+ACCUMULO_HOME=${ACCUMULO_HOME:-"${dir}/../../.."}
+ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-"$ACCUMULO_HOME/conf"}
+
+# Get the configuration values
+. ./merkle-env.sh
+
+tmpdir=$(mktemp -dt "$0.XXXXXXXXXX")
+
+splits=${tmpdir}/splits
+
+echo 1 >> $splits
+echo 2 >> $splits
+echo 3 >> $splits
+echo 4 >> $splits
+echo 5 >> $splits
+echo 6 >> $splits
+echo 7 >> $splits
+echo 8 >> $splits
+echo 9 >> $splits
+
+commands=${tmpdir}/commands
+
+# Generate leaves of merkle trees for source
+echo "deletetable -f $SOURCE_MERKLE_TABLE_NAME" >> $commands
+echo "createtable $SOURCE_MERKLE_TABLE_NAME" >> $commands
+echo "quit" >> $commands
+
+echo $SOURCE_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $SOURCE_ACCUMULO_USER -z \
+ $SOURCE_INSTANCE $SOURCE_ZOOKEEPERS -f $commands
+
+echo -e "\nGenerating merkle tree hashes for $SOURCE_TABLE_NAME"
+
+$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.cli.GenerateHashes -t $SOURCE_TABLE_NAME \
+ -o $SOURCE_MERKLE_TABLE_NAME -i $SOURCE_INSTANCE -z $SOURCE_ZOOKEEPERS -u $SOURCE_ACCUMULO_USER \
+ -p $SOURCE_ACCUMULO_PASSWORD -nt 8 -hash MD5 --splits $splits
+
+rm $commands
+
+# Generate leaves of merkle trees for destination
+echo "deletetable -f $DESTINATION_MERKLE_TABLE_NAME" >> $commands
+echo "createtable $DESTINATION_MERKLE_TABLE_NAME" >> $commands
+echo "quit" >> $commands
+
+echo $DESTINATION_ACCUMULO_PASSWORD | ${ACCUMULO_HOME}/bin/accumulo shell -u $DESTINATION_ACCUMULO_USER -z \
+ $DESTINATION_INSTANCE $DESTINATION_ZOOKEEPERS -f $commands
+
+echo -e "\nGenerating merkle tree hashes for $DESTINATION_TABLE_NAME"
+
+$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.cli.GenerateHashes -t $DESTINATION_TABLE_NAME \
+ -o $DESTINATION_MERKLE_TABLE_NAME -i $DESTINATION_INSTANCE -z $DESTINATION_ZOOKEEPERS -u $DESTINATION_ACCUMULO_USER \
+ -p $DESTINATION_ACCUMULO_PASSWORD -nt 8 -hash MD5 --splits $splits
+
+echo -e "\nComputing root hash:"
+
+#Compute root node of merkle tree
+$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.cli.ComputeRootHash -t $SOURCE_MERKLE_TABLE_NAME \
+ -i $SOURCE_INSTANCE -z $SOURCE_ZOOKEEPERS -u $SOURCE_ACCUMULO_USER -p $SOURCE_ACCUMULO_PASSWORD -hash MD5
+
+$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.replication.merkle.cli.ComputeRootHash -t $DESTINATION_MERKLE_TABLE_NAME \
+ -i $DESTINATION_INSTANCE -z $DESTINATION_ZOOKEEPERS -u $DESTINATION_ACCUMULO_USER -p $DESTINATION_ACCUMULO_PASSWORD -hash MD5
+
+rm -rf $tmpdir
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/scalability/README.md
----------------------------------------------------------------------
diff --git a/test/scalability/README.md b/test/scalability/README.md
new file mode 100644
index 0000000..4f287d2
--- /dev/null
+++ b/test/scalability/README.md
@@ -0,0 +1,57 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Apache Accumulo Scalability Tests
+=================================
+
+The scalability test framework needs to be configured for your Accumulo
+instance by performing the following steps.
+
+WARNING: Each scalability test rewrites your `conf/tservers` file and reinitializes
+your Accumulo instance. Do not run these tests on a cluster holding essential
+data.
+
+1. Make sure you have both `ACCUMULO_HOME` and `HADOOP_HOME` set in your
+ `$ACCUMULO_CONF_DIR/accumulo-env.sh.`
+
+2. Create a 'site.conf' file in the `conf` directory containing settings
+ needed by test nodes to connect to Accumulo, and to guide the tests.
+
+ `$ cp conf/site.conf.example conf/site.conf`
+
+3. Create an 'Ingest.conf' file in the `conf` directory containing performance
+ settings for the Ingest test. (This test is currently the only scalability
+ test available.)
+
+ `$ cp conf/Ingest.conf.example conf/Ingest.conf`
+
+ Each test has a unique ID (e.g., "Ingest") which correlates with its test
+ code in:
+
+ `org.apache.accumulo.test.scalability.tests.<ID>`
+
+ This ID correlates with a config file:
+
+ `conf/<ID>.conf`
+
+To run the test, specify its ID to the run.py script.
+
+> `$ nohup ./run.py Ingest > test1.log 2>&1 &`
+
+A timestamped directory will be created, and results are placed in it as each
+test completes.
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/scalability/conf/Ingest.conf.example
----------------------------------------------------------------------
diff --git a/test/scalability/conf/Ingest.conf.example b/test/scalability/conf/Ingest.conf.example
new file mode 100644
index 0000000..38800a1
--- /dev/null
+++ b/test/scalability/conf/Ingest.conf.example
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Table to ingest into
+TABLE=ingest
+# Number of entries to ingest
+NUM_ENTRIES=3000000
+
+# Batch writer configuration
+# Max memory (in bytes) that each ingester will use to buffer writes
+MAX_MEMORY=100000000
+# Max time (in millis) that each ingester will buffer data
+MAX_LATENCY=600000
+# Number of threads each ingester will use to write data
+NUM_THREADS=4
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/scalability/conf/site.conf.example
----------------------------------------------------------------------
diff --git a/test/scalability/conf/site.conf.example b/test/scalability/conf/site.conf.example
new file mode 100644
index 0000000..e1ce08f
--- /dev/null
+++ b/test/scalability/conf/site.conf.example
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# accumulo instance
+INSTANCE_NAME=a12
+# zookeepers
+ZOOKEEPERS=localhost
+# username
+USER=root
+# password
+PASSWORD=secret
+# used to determine which nodes to run tests on
+TSERVERS=/accumulo/conf/tservers
+# sets the number of nodes for each test case
+TEST_CASES=1,2,4,8,16
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/scalability/run.py
----------------------------------------------------------------------
diff --git a/test/scalability/run.py b/test/scalability/run.py
new file mode 100755
index 0000000..2b21a12
--- /dev/null
+++ b/test/scalability/run.py
@@ -0,0 +1,228 @@
+#!/usr/bin/python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import time
+import os
+import sys
+from ConfigParser import ConfigParser
+from subprocess import Popen, PIPE
+
+class JavaConfig:
+ '''Enable access to properities in java siteConfig file'''
+ def __init__(self, fname):
+ self.prop_d = {}
+ for line in open(fname):
+ line = line.strip();
+ if line.startswith('#') or len(line) == 0:
+ continue
+ pair = line.split('=')
+ if len(pair) != 2:
+ log.error("Invalid property (%s)" % line)
+ continue
+ self.prop_d[pair[0].strip()] = pair[1].strip()
+
+ def get(self, prop):
+ return self.prop_d[prop]
+
+def file_len(fname):
+ i=0
+ for line in open(fname):
+ i += 1
+ return i
+
+def runTest(testName, siteConfig, testDir, numNodes, fdata):
+
+ log('Stopping accumulo')
+ syscall('$ACCUMULO_HOME/bin/accumulo-cluster stop')
+
+ log('Creating tservers file for this test')
+ tserversPath = siteConfig.get('TSERVERS')
+ nodesPath = testDir+'/nodes/%d' % numNodes
+ syscall('head -n %d %s > %s' % (numNodes,tserversPath,nodesPath))
+
+ log('Copying tservers file to accumulo config')
+ syscall('cp '+nodesPath+' $ACCUMULO_CONF_DIR/tservers');
+
+ log('Removing /accumulo directory in HDFS')
+ syscall("hadoop fs -rmr /accumulo")
+
+ log('Initializing new Accumulo instance')
+ instance = siteConfig.get('INSTANCE_NAME')
+ passwd = siteConfig.get('PASSWORD')
+ syscall('printf "%s\nY\n%s\n%s\n" | $ACCUMULO_HOME/bin/accumulo init' % (instance, passwd, passwd))
+
+ log('Starting new Accumulo instance')
+ syscall('$ACCUMULO_HOME/bin/accumulo-cluster start')
+
+ sleepTime = 30
+ if numNodes > 120:
+ sleepTime = int(numNodes / 4)
+ log('Sleeping for %d seconds' % sleepTime)
+ time.sleep(sleepTime)
+
+ log('Setting up %s test' % testName)
+ syscall('$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.scalability.Run %s setup %s' % (testName, numNodes))
+
+ log('Sleeping for 5 seconds')
+ time.sleep(5)
+
+ log('Starting %s clients' % testName)
+ numThreads = numNodes
+ if int(numNodes) > 128:
+ numThreads='128'
+ syscall('pssh -P -h %s -p %s "$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.scalability.Run %s client %s >/tmp/scale.out 2>/tmp/scale.err &" < /dev/null' % (nodesPath, numThreads, testName, numNodes))
+
+ log('Sleeping for 30 sec before checking how many clients started...')
+ time.sleep(30)
+ output = Popen(["hadoop fs -ls /accumulo-scale/clients"], stdout=PIPE, shell=True).communicate()[0]
+ num_clients = int(output.split()[1])
+ log('%s clients started!' % num_clients)
+
+ log('Waiting until %d clients finish.' % num_clients)
+ last = 0
+ done = 0
+ while done < num_clients:
+ time.sleep(5)
+ output = Popen(["hadoop fs -ls /accumulo-scale/results"], stdout=PIPE, shell=True).communicate()[0]
+ if not output:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+ continue
+ done = int(output.split()[1])
+ if done != last:
+ sys.stdout.write('.%s' % done)
+ else:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+ last = done
+ sys.stdout.flush()
+ log('\nAll clients are finished!')
+
+ log('Copying results from HDFS')
+ resultsDir = "%s/results/%s" % (testDir, numNodes)
+ syscall('hadoop fs -copyToLocal /accumulo-scale/results %s' % resultsDir)
+
+ log('Calculating results from clients')
+ times = []
+ totalMs = 0L
+ totalEntries = 0L
+ totalBytes = 0L
+ for fn in os.listdir(resultsDir):
+ for line in open('%s/%s' % (resultsDir,fn)):
+ words = line.split()
+ if words[0] == 'ELAPSEDMS':
+ ms = long(words[1].strip())
+ totalMs += ms
+ times.append(ms)
+ totalEntries += long(words[2].strip())
+ totalBytes += long(words[3].strip())
+ times.sort()
+
+ print times
+ numClients = len(times)
+ min = times[0] / 1000
+ avg = (float(totalMs) / numClients) / 1000
+ median = times[int(numClients/2)] / 1000
+ max = times[numClients-1] / 1000
+ q1 = times[int(numClients/4)] / 1000
+ q3 = times[int((3*numClients)/4)] / 1000
+
+ log('Tservs\tClients\tMin\tQ1\tAvg\tMed\tQ3\tMax\tEntries\tMB')
+ resultStr = '%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%dM\t%d' % (numNodes, numClients, min, q1, avg, median, q3, max, totalEntries / 1000000, totalBytes / 1000000)
+ log(resultStr)
+ fdata.write(resultStr + '\n')
+ fdata.flush()
+
+ time.sleep(5)
+
+ log('Tearing down %s test' % testName)
+ syscall('$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.test.scalability.Run %s teardown %s' % (testName, numNodes))
+
+ time.sleep(10)
+
+def syscall(cmd):
+ log('> %s' % cmd)
+ os.system(cmd)
+
+def run(cmd, **kwargs):
+ log.debug("Running %s", ' '.join(cmd))
+ handle = Popen(cmd, stdout=PIPE, **kwargs)
+ out, err = handle.communicate()
+ log.debug("Result %d (%r, %r)", handle.returncode, out, err)
+ return handle.returncode
+
+def log(msg):
+ print msg
+ sys.stdout.flush()
+
+def main():
+
+ if not os.getenv('ACCUMULO_HOME'):
+ raise 'ACCUMULO_HOME needs to be set!'
+
+ if not os.getenv('ACCUMULO_CONF_DIR'):
+ raise 'ACCUMULO_CONF_DIR needs to be set!'
+
+ if not os.getenv('HADOOP_HOME'):
+ raise 'HADOOP_HOME needs to be set!'
+
+ if len(sys.argv) != 2:
+ log('Usage: run.py <testName>')
+ sys.exit()
+ testName = sys.argv[1]
+
+ logging.basicConfig(level=logging.DEBUG)
+
+ log('Creating test directory structure')
+ testDir = 'test-%d' % time.time()
+ nodesDir = testDir+'/nodes'
+ resultsDir = testDir+'/results'
+ syscall('mkdir %s' % testDir)
+ syscall('mkdir %s' % nodesDir)
+ syscall('mkdir %s' % resultsDir)
+
+ log('Removing current /accumulo-scale directory')
+ syscall('hadoop fs -rmr /accumulo-scale')
+
+ log('Creating new /accumulo-scale directory structure')
+ syscall('hadoop fs -mkdir /accumulo-scale')
+ syscall('hadoop fs -mkdir /accumulo-scale/clients')
+ syscall('hadoop fs -mkdir /accumulo-scale/results')
+ syscall('hadoop fs -chmod -R 777 /accumulo-scale')
+
+ log('Copying config to HDFS')
+ syscall('hadoop fs -copyFromLocal ./conf /accumulo-scale/conf')
+
+ siteConfig = JavaConfig('conf/site.conf');
+ tserversPath = siteConfig.get('TSERVERS')
+ maxNodes = file_len(tserversPath)
+
+ fdata = open('%s/scale.dat' % testDir, 'w')
+ fdata.write('Tservs\tClients\tMin\tAvg\tMed\tMax\tEntries\tMB\n')
+
+ for numNodes in siteConfig.get('TEST_CASES').split(','):
+ log('Running %s test with %s nodes' % (testName, numNodes))
+ if int(numNodes) > maxNodes:
+ logging.error('Skipping %r test case as tservers file %r contains only %r nodes', numNodes, tserversPath, maxNodes)
+ continue
+ runTest(testName, siteConfig, testDir, int(numNodes), fdata)
+ sys.stdout.flush()
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/scale/agitator.txt
----------------------------------------------------------------------
diff --git a/test/scale/agitator.txt b/test/scale/agitator.txt
new file mode 100644
index 0000000..9715bf0
--- /dev/null
+++ b/test/scale/agitator.txt
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+* run continuous ingest, continuous walkers, and agitator for 24 hours
+* stop continuous ingest, walkers, and agitator
+* run verification map reduce job
+
+This test should be run with the following configurations of the agitator.
+* agitator does not kill master and only kills one tablet server at a time
+* agitator does not kill master and only kills multiple tablet server at a time
+* agitator kills master and only kills multiple tablet server at a time
+* agitator kills master and does not kill tablet servers
+
+The agitator can also be run during verification. Another test is to lower
+a tables split threshold during verification.
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/scale/catastrophic.txt
----------------------------------------------------------------------
diff --git a/test/scale/catastrophic.txt b/test/scale/catastrophic.txt
new file mode 100644
index 0000000..322cab7
--- /dev/null
+++ b/test/scale/catastrophic.txt
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+* run continuous ingest and continuous walkers for 24 hours
+* stop continuous ingest and walkers
+* disable idle compactions
+* run verification map reduce job
+* ensure there are entries in memory and then kill -9 all tablet servers and master
+* restart accumulo
+* allow log recoveries to run
+* run verification map reduce job
+* compare counts from two verification runs
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/scale/deleteLargeTable.txt
----------------------------------------------------------------------
diff --git a/test/scale/deleteLargeTable.txt b/test/scale/deleteLargeTable.txt
new file mode 100644
index 0000000..59dd648
--- /dev/null
+++ b/test/scale/deleteLargeTable.txt
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Create and delete a large table (do this with accumulo.metadata split alot) see #2381
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/scale/restart.txt
----------------------------------------------------------------------
diff --git a/test/scale/restart.txt b/test/scale/restart.txt
new file mode 100644
index 0000000..bab7cd9
--- /dev/null
+++ b/test/scale/restart.txt
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ * create a continuous ingest table with 16k tablets
+ * start continuous ingest
+ * restart accumulo, but leave the ingesters running
+ * after accumulo is up, ingest should quickly resume to pre-restart rates
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/README.md
----------------------------------------------------------------------
diff --git a/test/stress/README.md b/test/stress/README.md
new file mode 100644
index 0000000..9eb9fdc
--- /dev/null
+++ b/test/stress/README.md
@@ -0,0 +1,105 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Accumulo Stress Test
+====================
+
+This is an attempt to observe the behavior Accumulo displays when compacting and
+reading cells. There are two components to this package:
+`org.apache.accumulo.test.stress.random.Write` and
+`org.apache.accumulo.test.stress.random.Scan`.
+
+Since this test is meant to stress the system, there is no pre-defined test
+period and no success criteria that we can programmatically check. During a
+successful test period, the Accumulo cluster should remain stable and
+functional. Possible failure conditions include component failures, such as
+tablet servers running out of memory.
+
+The `org.apache.accumulo.test.stress.random.Write` utility provides facilities
+for writing random sized cells. Users can configure minimum and maximum sized
+portions of a cell. The portions users can configure are the row, column family,
+column qualifier and value. Note that the sizes are uniformly distributed
+between the minimum and maximum values.
+
+The `org.apache.accumulo.test.stress.random.Scan` utility provides users with
+the ability to query tables generated by the Write. It will pick a tablet at
+random and scan the entire range. The amount of times this process is performed
+is user configurable. By default, it happens 1,024 times. Users can also specify
+whether or not the scan should be isolated.
+
+There is no shared state intended by either of these services. This allows
+multiple clients to be run in parallel, either on the same host or distributed
+across hosts.
+
+## Prerequisites
+
+The utilities are run on a normal, initialized Accumulo cluster. By default,
+they only work with a table named "stress_test".
+
+The start and stop scripts rely on pssh. Before running them, you will need
+to install pssh on the machines that will be controlled by them.
+
+## Running
+
+Copy `stress-env.sh.example` to `stress-env.sh` and edit it, supplying at least
+correct values for the USERPASS and INSTANCE variables. The remainder of this
+file describes and provides defaults for many of the configuration options
+for the stress utilities.
+
+### Individual Execution
+
+Before starting a fresh write test, clear out the test table using the Accumulo
+shell.
+
+ > deletetable -f stress_test
+ > createtable stress_test
+
+To run a writer:
+
+ $ ./writer.sh
+
+The writer will begin logging into a "logs" subdirectory, using timestamped
+filenames. You can stop the writer by killing it or using Control-C.
+
+To run a reader:
+
+ $ ./reader.sh
+
+The reader logs like the writer. By default, the reader reads forever, but
+it can be configured for a fixed number of scan iterations.
+
+### Group Execution
+
+To run writers across multiple machines, create a writers file and list each
+host that will run a writer, one per line.
+
+Finally, to start and stop the writers:
+
+ $ ./start-writers.sh
+ $ ./stop-writers.sh
+
+For readers, make a similar readers file, and use `start-readers.sh` and
+`stop-readers.sh`.
+
+The start scripts copy out stress-env.sh to each of the writers and readers,
+overwriting what is there, so you only need to edit the file on the host from
+which you run those scripts.
+
+Note that in versions of Accumulo greater than 1.6.0, users should verify that
+the cell sizes they supply meet the criteria set forth by the key constraints
+on their tables. By default, tables are configured to reject keys that are
+greater than 1MB.
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/reader.sh
----------------------------------------------------------------------
diff --git a/test/stress/reader.sh b/test/stress/reader.sh
new file mode 100755
index 0000000..e282632
--- /dev/null
+++ b/test/stress/reader.sh
@@ -0,0 +1,39 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start: Resolve Script Directory
+# Ref: http://stackoverflow.com/questions/59895/
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+ SOURCE=$(readlink "${SOURCE}")
+ [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+# Stop: Resolve Script Directory
+LOG_DIR=${DIR}/logs
+mkdir -p "$LOG_DIR"
+
+# Source environment
+. "${DIR}/stress-env.sh"
+
+ts=$(date +%Y%m%d%H%M%S)
+host=$(hostname)
+# We want USERPASS to word split
+"${ACCUMULO_HOME}/bin/accumulo org.apache.accumulo.test.stress.random.Scan" "$INSTANCE" $USERPASS "$SCAN_SEED" "$CONTINUOUS_SCAN" "$SCAN_BATCH_SIZE" \
+ > "$LOG_DIR/${ts}_${host}_reader.out" \
+ 2> "$LOG_DIR/${ts}_${host}_reader.err"
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/readers
----------------------------------------------------------------------
diff --git a/test/stress/readers b/test/stress/readers
new file mode 100644
index 0000000..80a4224
--- /dev/null
+++ b/test/stress/readers
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+readerhost1
+readerhost2
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/start-readers.sh
----------------------------------------------------------------------
diff --git a/test/stress/start-readers.sh b/test/stress/start-readers.sh
new file mode 100755
index 0000000..6f651f4
--- /dev/null
+++ b/test/stress/start-readers.sh
@@ -0,0 +1,40 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start: Resolve Script Directory
+# Ref: http://stackoverflow.com/questions/59895/
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR="$( cd -P "$( dirname "${SOURCE}" )" && pwd )"
+ SOURCE="$(readlink "${SOURCE}")"
+ [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "${SOURCE}" )" && pwd )"
+# Stop: Resolve Script Directory
+
+# Source environment
+. "${DIR}/stress-env.sh"
+
+if [[ ! -f ${DIR}/readers ]]; then
+ echo readers file is missing
+ exit 1
+fi
+
+# Copy environment out
+pscp -h "${DIR}/readers" "${DIR}/stress-env.sh" "${DIR}"
+
+pssh -h "${DIR}/readers" "nohup ${DIR}/reader.sh >${DIR}/reader.out 2>${DIR}/reader.err < /dev/null &"
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/start-writers.sh
----------------------------------------------------------------------
diff --git a/test/stress/start-writers.sh b/test/stress/start-writers.sh
new file mode 100755
index 0000000..474117c
--- /dev/null
+++ b/test/stress/start-writers.sh
@@ -0,0 +1,40 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start: Resolve Script Directory
+# Ref: http://stackoverflow.com/questions/59895/
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+ SOURCE=$(readlink "${SOURCE}")
+ [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+# Stop: Resolve Script Directory
+
+# Source environment
+. "${DIR}/stress-env.sh"
+
+if [[ ! -f ${DIR}/writers ]]; then
+ echo writers file is missing
+ exit 1
+fi
+
+# Copy environment out
+pscp -h "${DIR}/writers" "${DIR}/stress-env.sh" "${DIR}"
+
+pssh -h "${DIR}/writers" "nohup ${DIR}/writer.sh >${DIR}/writer.out 2>${DIR}/writer.err < /dev/null &"
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/stop-readers.sh
----------------------------------------------------------------------
diff --git a/test/stress/stop-readers.sh b/test/stress/stop-readers.sh
new file mode 100755
index 0000000..8bd4a6f
--- /dev/null
+++ b/test/stress/stop-readers.sh
@@ -0,0 +1,36 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start: Resolve Script Directory
+# Ref: http://stackoverflow.com/questions/59895/
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+ SOURCE=$(readlink "${SOURCE}")
+ [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+# Stop: Resolve Script Directory
+
+# Source environment
+. "${DIR}/stress-env.sh"
+
+if [[ ! -f "${DIR}/readers" ]]; then
+ echo readers file is missing
+ exit 1
+fi
+pssh -h "${DIR}/readers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Scan'" < /dev/null
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/stop-writers.sh
----------------------------------------------------------------------
diff --git a/test/stress/stop-writers.sh b/test/stress/stop-writers.sh
new file mode 100755
index 0000000..5c9e8d7
--- /dev/null
+++ b/test/stress/stop-writers.sh
@@ -0,0 +1,36 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start: Resolve Script Directory
+# Ref: http://stackoverflow.com/questions/59895/
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+ SOURCE=$(readlink "${SOURCE}")
+ [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+# Stop: Resolve Script Directory
+
+# Source environment
+. "${DIR}/stress-env.sh"
+
+if [[ ! -f "${DIR}/writers" ]]; then
+ echo writers file is missing
+ exit 1
+fi
+pssh -h "${DIR}/writers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Write'" < /dev/null
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/stress-env.sh.example
----------------------------------------------------------------------
diff --git a/test/stress/stress-env.sh.example b/test/stress/stress-env.sh.example
new file mode 100644
index 0000000..6efb24f
--- /dev/null
+++ b/test/stress/stress-env.sh.example
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ACCUMULO_HOME=${ACCUMULO_HOME:-/opt/accumulo}
+
+# Edit the credentials to match your system
+USERPASS='-u root -p secret'
+INSTANCE='-z localhost -i inst'
+
+### Read configuration ###
+
+# This is the seed for the range picking logic used by the scanner.
+SCAN_SEED='--scan-seed 1337'
+
+# Controls the number of random tablets the scanner will read sequentially
+#SCAN_ITERATIONS='--num-iterations 1024'
+
+# Alternatively, we can just continously scan
+CONTINUOUS_SCAN='--continuous'
+
+# Controls whether or not the scan will be using an isolated scanner. Add this to the execution
+#SCAN_ISOLATION='--isolate'
+
+# Sets the batch size for the scanner, use a lower number for large rows / cells
+#SCAN_BATCH_SIZE='--scan-batch-size -1'
+
+### Write configuration ###
+
+# Edit these to change the range of each cell component. The size is in bytes.
+ROW_RANGE='--min-row-size 128 --max-row-size 128'
+CF_RANGE='--min-cf-size 128 --max-cf-size 128'
+CQ_RANGE='--min-cq-size 128 --max-cq-size 128'
+VALUE_RANGE='--min-value-size 1024 --max-value-size 2048'
+ROW_WIDTH='--min-row-width 1 --max-row-width 10'
+
+# These are the seeds for the random number generates used to generate each cell component.
+ROW_SEED='--row-seed 1'
+CF_SEED='--cf-seed 2'
+CQ_SEED='--cq-seed 3'
+VALUE_SEED='--value-seed 4'
+ROW_WIDTH_SEED='--row-width-seed 5'
+
+# This is the maximum number of cells to include in each mutation written out.
+# A non-positive value implies no limit.
+MAX_CELLS_PER_MUTATION='--max-cells-per-mutation -1'
+
+# This is the delay in milliseconds between writes. Use <= 0 for no delay.
+WRITE_DELAY='--write-delay 0'
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/writer.sh
----------------------------------------------------------------------
diff --git a/test/stress/writer.sh b/test/stress/writer.sh
new file mode 100755
index 0000000..7107242
--- /dev/null
+++ b/test/stress/writer.sh
@@ -0,0 +1,44 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Start: Resolve Script Directory
+# Ref: http://stackoverflow.com/questions/59895/
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+ SOURCE=$(readlink "${SOURCE}")
+ [[ "${SOURCE}" != /* ]] && SOURCE="${DIR}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR=$( cd -P "$( dirname "${SOURCE}" )" && pwd )
+# Stop: Resolve Script Directory
+LOG_DIR=${DIR}/logs
+mkdir -p "$LOG_DIR"
+
+# Source environment
+. "${DIR}/stress-env.sh"
+
+ts=$(date +%Y%m%d%H%M%S)
+host=$(hostname)
+
+# TBD - --clear-table option
+
+# We want $USERPASS to word split
+"${ACCUMULO_HOME}/bin/accumulo org.apache.accumulo.test.stress.random.Write" "$INSTANCE" $USERPASS "$ROW_RANGE" "$CF_RANGE" "$CQ_RANGE" "$VALUE_RANGE" \
+ "$ROW_SEED" "$CF_SEED" "$CQ_SEED" "$VALUE_SEED" \
+ "$ROW_WIDTH" "$ROW_WIDTH_SEED" "$MAX_CELLS_PER_MUTATION" "$WRITE_DELAY" \
+ > "$LOG_DIR/${ts}_${host}_writer.out" \
+ 2> "$LOG_DIR/${ts}_${host}_writer.err"
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/stress/writers
----------------------------------------------------------------------
diff --git a/test/stress/writers b/test/stress/writers
new file mode 100644
index 0000000..0a06590
--- /dev/null
+++ b/test/stress/writers
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+writerhost1
+writerhost2
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/test1/README.md
----------------------------------------------------------------------
diff --git a/test/test1/README.md b/test/test1/README.md
new file mode 100644
index 0000000..00767b7
--- /dev/null
+++ b/test/test1/README.md
@@ -0,0 +1,46 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Command to run from command line
+
+Can run this test with pre-existing splits; use the following command to create the table with
+100 pre-existing splits
+
+> `$ ../../../bin/accumulo 'org.apache.accumulo.test.TestIngest' --createTable \
+-u root -p secret --splits 100 --rows 0`
+
+Could try running verify commands after stopping and restarting accumulo
+
+When write ahead log is implemented can try killing tablet server in middle of ingest
+
+Run 5 parallel ingesters and verify:
+
+> `$ . ingest_test.sh`
+(wait)
+`$ . verify_test.sh`
+(wait)
+
+Overwrite previous ingest:
+> `$ . ingest_test_2.sh`
+(wait)
+`$ . verify_test_2.sh`
+(wait)
+
+Delete what was previously ingested:
+> `$ . ingest_test_3.sh`
+(wait)
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/test1/ingest_test.sh
----------------------------------------------------------------------
diff --git a/test/test1/ingest_test.sh b/test/test1/ingest_test.sh
new file mode 100755
index 0000000..65bf2bd
--- /dev/null
+++ b/test/test1/ingest_test.sh
@@ -0,0 +1,22 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 0 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 1000000 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 2000000 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 3000000 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 1 --size 50 --random 56 --rows 1000000 --start 4000000 --cols 1 &
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/test1/ingest_test_2.sh
----------------------------------------------------------------------
diff --git a/test/test1/ingest_test_2.sh b/test/test1/ingest_test_2.sh
new file mode 100755
index 0000000..c1c5f75
--- /dev/null
+++ b/test/test1/ingest_test_2.sh
@@ -0,0 +1,22 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 0 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 1000000 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 2000000 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 3000000 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 2 --size 50 --random 57 --rows 1000000 --start 4000000 --cols 1 &
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/test1/ingest_test_3.sh
----------------------------------------------------------------------
diff --git a/test/test1/ingest_test_3.sh b/test/test1/ingest_test_3.sh
new file mode 100755
index 0000000..650e3fe
--- /dev/null
+++ b/test/test1/ingest_test_3.sh
@@ -0,0 +1,22 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 0 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 1000000 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 2000000 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 3000000 --cols 1 &
+../../../bin/accumulo org.apache.accumulo.test.TestIngest -u root -p secret --timestamp 3 --delete --rows 1000000 --start 4000000 --cols 1 &