You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/01/25 18:21:53 UTC
[4/5] accumulo-testing git commit: ACCUMULO-4510 Moved remaining
external test code from Accumulo
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java
new file mode 100644
index 0000000..87489f6
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.merkle.cli;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.cli.ClientOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Accepts two table names and enumerates all key-values pairs in both checking for correctness. All differences between the two tables will be printed to the
+ * console.
+ */
+public class ManualComparison {
+
+ public static class ManualComparisonOpts extends ClientOpts {
+ @Parameter(names = {"--table1"}, required = true, description = "First table")
+ public String table1;
+
+ @Parameter(names = {"--table2"}, required = true, description = "First table")
+ public String table2;
+ }
+
+ public static void main(String[] args) throws Exception {
+ ManualComparisonOpts opts = new ManualComparisonOpts();
+ opts.parseArgs("ManualComparison", args);
+
+ Connector conn = opts.getConnector();
+
+ Scanner s1 = conn.createScanner(opts.table1, Authorizations.EMPTY), s2 = conn.createScanner(opts.table2, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> iter1 = s1.iterator(), iter2 = s2.iterator();
+ boolean incrementFirst = true, incrementSecond = true;
+
+ Entry<Key,Value> entry1 = iter1.next(), entry2 = iter2.next();
+ while (iter1.hasNext() && iter2.hasNext()) {
+ if (incrementFirst) {
+ entry1 = iter1.next();
+ }
+ if (incrementSecond) {
+ entry2 = iter2.next();
+ }
+ incrementFirst = false;
+ incrementSecond = false;
+
+ if (!entry1.equals(entry2)) {
+
+ if (entry1.getKey().compareTo(entry2.getKey()) < 0) {
+ System.out.println("Exist in original " + entry1);
+ incrementFirst = true;
+ } else if (entry2.getKey().compareTo(entry1.getKey()) < 0) {
+ System.out.println("Exist in replica " + entry2);
+ incrementSecond = true;
+ } else {
+ System.out.println("Differ... " + entry1 + " " + entry2);
+ incrementFirst = true;
+ incrementSecond = true;
+ }
+ } else {
+ incrementFirst = true;
+ incrementSecond = true;
+ }
+ }
+
+ System.out.println("\nExtra entries from " + opts.table1);
+ while (iter1.hasNext()) {
+ System.out.println(iter1.next());
+ }
+
+ System.out.println("\nExtra entries from " + opts.table2);
+ while (iter2.hasNext()) {
+ System.out.println(iter2.next());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java
new file mode 100644
index 0000000..440b009
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.merkle.ingest;
+
+import java.util.Random;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Generates some random data with a given percent of updates to be deletes.
+ */
+public class RandomWorkload {
+ public static final String DEFAULT_TABLE_NAME = "randomWorkload";
+
+ public static class RandomWorkloadOpts extends ClientOnDefaultTable {
+ @Parameter(names = {"-n", "--num"}, required = true, description = "Num records to write")
+ public long numRecords;
+
+ @Parameter(names = {"-r", "--rows"}, required = true, description = "Range of rows that can be generated")
+ public int rowMax;
+
+ @Parameter(names = {"-cf", "--colfams"}, required = true, description = "Range of column families that can be generated")
+ public int cfMax;
+
+ @Parameter(names = {"-cq", "--colquals"}, required = true, description = "Range of column qualifiers that can be generated")
+ public int cqMax;
+
+ @Parameter(names = {"-d", "--deletes"}, required = false, description = "Percentage of updates that should be deletes")
+ public int deletePercent = 5;
+
+ public RandomWorkloadOpts() {
+ super(DEFAULT_TABLE_NAME);
+ }
+
+ public RandomWorkloadOpts(String tableName) {
+ super(tableName);
+ }
+ }
+
+ public void run(RandomWorkloadOpts opts, BatchWriterConfig cfg) throws Exception {
+ run(opts.getConnector(), opts.getTableName(), cfg, opts.numRecords, opts.rowMax, opts.cfMax, opts.cqMax, opts.deletePercent);
+ }
+
+ public void run(final Connector conn, final String tableName, final BatchWriterConfig cfg, final long numRecords, int rowMax, int cfMax, int cqMax,
+ int deletePercent) throws Exception {
+
+ final Random rowRand = new Random(12345);
+ final Random cfRand = new Random(12346);
+ final Random cqRand = new Random(12347);
+ final Random deleteRand = new Random(12348);
+ long valueCounter = 0l;
+
+ if (!conn.tableOperations().exists(tableName)) {
+ conn.tableOperations().create(tableName);
+ }
+
+ BatchWriter bw = conn.createBatchWriter(tableName, cfg);
+ try {
+ final Text row = new Text(), cf = new Text(), cq = new Text();
+ final Value value = new Value();
+ for (long i = 0; i < numRecords; i++) {
+ row.set(Integer.toString(rowRand.nextInt(rowMax)));
+ cf.set(Integer.toString(cfRand.nextInt(cfMax)));
+ cq.set(Integer.toString(cqRand.nextInt(cqMax)));
+
+ Mutation m = new Mutation(row);
+
+ // Choose a random value between [0,100)
+ int deleteValue = deleteRand.nextInt(100);
+
+ // putDelete if the value we chose is less than our delete percentage
+ if (deleteValue < deletePercent) {
+ m.putDelete(cf, cq);
+ } else {
+ value.set(Long.toString(valueCounter).getBytes());
+ m.put(cf, cq, valueCounter, value);
+ }
+
+ bw.addMutation(m);
+
+ valueCounter++;
+ }
+ } finally {
+ bw.close();
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ RandomWorkloadOpts opts = new RandomWorkloadOpts();
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+ opts.parseArgs(RandomWorkload.class.getSimpleName(), args, bwOpts);
+
+ RandomWorkload rw = new RandomWorkload();
+
+ rw.run(opts, bwOpts.getBatchWriterConfig());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/merkle/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/package-info.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/package-info.java
new file mode 100644
index 0000000..5b27e4b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/package-info.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * A <a href="http://en.wikipedia.org/wiki/Merkle_tree">Merkle tree</a> is a hash tree and can be used to evaluate equality over large
+ * files with the ability to ascertain what portions of the files differ. Each leaf of the Merkle tree is some hash of a
+ * portion of the file, with each leaf corresponding to some "range" within the source file. As such, if all leaves are
+ * considered as ranges of the source file, the "sum" of all leaves creates a contiguous range over the entire file.
+ * <p>
+ * The parent of any nodes (typically, a binary tree; however this is not required) is the concatenation of the hashes of
+ * the children. We can construct a full tree by walking up the tree, creating parents from children, until we have a root
+ * node. To check equality of two files that each have a merkle tree built, we can very easily compare the value of at the
+ * root of the Merkle tree to know whether or not the files are the same.
+ * <p>
+ * Additionally, in the situation where we have two files with we expect to be the same but are not, we can walk back down
+ * the tree, finding subtrees that are equal and subtrees that are not. Subtrees that are equal correspond to portions of
+ * the files which are identical, where subtrees that are not equal correspond to discrepancies between the two files.
+ * <p>
+ * We can apply this concept to Accumulo, treating a table as a file, and ranges within a file as an Accumulo Range. We can
+ * then compute the hashes over each of these Ranges and compute the entire Merkle tree to determine if two tables are
+ * equivalent.
+ *
+ * @since 1.7.0
+ */
+package org.apache.accumulo.testing.core.merkle;
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/merkle/skvi/DigestIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/skvi/DigestIterator.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/skvi/DigestIterator.java
new file mode 100644
index 0000000..ab7ad43
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/skvi/DigestIterator.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testin.core.merkle.skvi;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+
+/**
+ * {@link SortedKeyValueIterator} which attempts to compute a hash over some range of Key-Value pairs.
+ * <p>
+ * For the purposes of constructing a Merkle tree, this class will only generate a meaningful result if the (Batch)Scanner will compute a single digest over a
+ * Range. If the (Batch)Scanner stops and restarts in the middle of a session, incorrect values will be returned and the merkle tree will be invalid.
+ */
+public class DigestIterator implements SortedKeyValueIterator<Key,Value> {
+ public static final String HASH_NAME_KEY = "hash.name";
+
+ private MessageDigest digest;
+ private Key topKey;
+ private Value topValue;
+ private SortedKeyValueIterator<Key,Value> source;
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ String hashName = options.get(HASH_NAME_KEY);
+ if (null == hashName) {
+ throw new IOException(HASH_NAME_KEY + " must be provided as option");
+ }
+
+ try {
+ this.digest = MessageDigest.getInstance(hashName);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException(e);
+ }
+
+ this.topKey = null;
+ this.topValue = null;
+ this.source = source;
+ }
+
+ @Override
+ public boolean hasTop() {
+ return null != topKey;
+ }
+
+ @Override
+ public void next() throws IOException {
+ // We can't call next() if we already consumed it all
+ if (!this.source.hasTop()) {
+ this.topKey = null;
+ this.topValue = null;
+ return;
+ }
+
+ this.source.next();
+
+ consume();
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ this.source.seek(range, columnFamilies, inclusive);
+
+ consume();
+ }
+
+ protected void consume() throws IOException {
+ digest.reset();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+
+ if (!this.source.hasTop()) {
+ this.topKey = null;
+ this.topValue = null;
+
+ return;
+ }
+
+ Key lastKeySeen = null;
+ while (this.source.hasTop()) {
+ baos.reset();
+
+ Key currentKey = this.source.getTopKey();
+ lastKeySeen = currentKey;
+
+ currentKey.write(dos);
+ this.source.getTopValue().write(dos);
+
+ digest.update(baos.toByteArray());
+
+ this.source.next();
+ }
+
+ this.topKey = lastKeySeen;
+ this.topValue = new Value(digest.digest());
+ }
+
+ @Override
+ public Key getTopKey() {
+ return topKey;
+ }
+
+ @Override
+ public Value getTopValue() {
+ return topValue;
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ DigestIterator copy = new DigestIterator();
+ try {
+ copy.digest = MessageDigest.getInstance(digest.getAlgorithm());
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+
+ copy.topKey = this.topKey;
+ copy.topValue = this.topValue;
+ copy.source = this.source.deepCopy(env);
+
+ return copy;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/scalability/Ingest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/scalability/Ingest.java b/core/src/main/java/org/apache/accumulo/testing/core/scalability/Ingest.java
new file mode 100644
index 0000000..1a61eea
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/scalability/Ingest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.scalability;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.testing.core.continuous.ContinuousIngest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Ingest extends ScaleTest {
+
+ private static final Logger log = LoggerFactory.getLogger(Ingest.class);
+
+ @Override
+ public void setup() {
+
+ Connector conn = getConnector();
+ String tableName = getTestProperty("TABLE");
+
+ // delete existing table
+ if (conn.tableOperations().exists(tableName)) {
+ System.out.println("Deleting existing table: " + tableName);
+ try {
+ conn.tableOperations().delete(tableName);
+ } catch (Exception e) {
+ log.error("Failed to delete table '" + tableName + "'.", e);
+ }
+ }
+
+ // create table
+ try {
+ conn.tableOperations().create(tableName);
+ conn.tableOperations().addSplits(tableName, calculateSplits());
+ conn.tableOperations().setProperty(tableName, "table.split.threshold", "256M");
+ } catch (Exception e) {
+ log.error("Failed to create table '" + tableName + "'.", e);
+ }
+
+ }
+
+ @Override
+ public void client() {
+
+ Connector conn = getConnector();
+ String tableName = getTestProperty("TABLE");
+
+ // get batch writer configuration
+ long maxMemory = Long.parseLong(getTestProperty("MAX_MEMORY"));
+ long maxLatency = Long.parseLong(getTestProperty("MAX_LATENCY"));
+ int maxWriteThreads = Integer.parseInt(getTestProperty("NUM_THREADS"));
+
+ // create batch writer
+ BatchWriter bw = null;
+ try {
+ bw = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(maxMemory).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS)
+ .setMaxWriteThreads(maxWriteThreads));
+ } catch (TableNotFoundException e) {
+ log.error("Table '" + tableName + "' not found.", e);
+ System.exit(-1);
+ }
+
+ // configure writing
+ Random r = new Random();
+ String ingestInstanceId = UUID.randomUUID().toString();
+ long numIngestEntries = Long.parseLong(getTestProperty("NUM_ENTRIES"));
+ long minRow = 0L;
+ long maxRow = 9223372036854775807L;
+ int maxColF = 32767;
+ int maxColQ = 32767;
+ long count = 0;
+ long totalBytes = 0;
+
+ ColumnVisibility cv = new ColumnVisibility();
+
+ // start timer
+ startTimer();
+
+ // write specified number of entries
+ while (count < numIngestEntries) {
+ count++;
+ long rowId = ContinuousIngest.genLong(minRow, maxRow, r);
+ Mutation m = ContinuousIngest.genMutation(rowId, r.nextInt(maxColF), r.nextInt(maxColQ), cv, ingestInstanceId.getBytes(UTF_8), count, null, false);
+ totalBytes += m.numBytes();
+ try {
+ bw.addMutation(m);
+ } catch (MutationsRejectedException e) {
+ log.error("Mutations rejected.", e);
+ System.exit(-1);
+ }
+ }
+
+ // close writer
+ try {
+ bw.close();
+ } catch (MutationsRejectedException e) {
+ log.error("Could not close BatchWriter due to mutations being rejected.", e);
+ System.exit(-1);
+ }
+
+ // stop timer
+ stopTimer(count, totalBytes);
+ }
+
+ @Override
+ public void teardown() {
+
+ Connector conn = getConnector();
+ String tableName = getTestProperty("TABLE");
+
+ try {
+ conn.tableOperations().delete(tableName);
+ } catch (Exception e) {
+ log.error("Failed to delete table '" + tableName + "'", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/scalability/Run.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/scalability/Run.java b/core/src/main/java/org/apache/accumulo/testing/core/scalability/Run.java
new file mode 100644
index 0000000..f7f7458
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/scalability/Run.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.scalability;
+
+import java.io.FileInputStream;
+import java.net.InetAddress;
+import java.util.Properties;
+
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.Parameter;
+
+public class Run {
+
+ private static final Logger log = LoggerFactory.getLogger(Run.class);
+
+ static class Opts extends Help {
+ @Parameter(names = "--testId", required = true)
+ String testId;
+ @Parameter(names = "--action", required = true, description = "one of 'setup', 'teardown' or 'client'")
+ String action;
+ @Parameter(names = "--count", description = "number of tablet servers", required = true)
+ int numTabletServers;
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ final String sitePath = "/tmp/scale-site.conf";
+ final String testPath = "/tmp/scale-test.conf";
+ Opts opts = new Opts();
+ opts.parseArgs(Run.class.getName(), args);
+
+ Configuration conf = CachedConfiguration.getInstance();
+ FileSystem fs;
+ fs = FileSystem.get(conf);
+
+ fs.copyToLocalFile(new Path("/accumulo-scale/conf/site.conf"), new Path(sitePath));
+ fs.copyToLocalFile(new Path(String.format("/accumulo-scale/conf/%s.conf", opts.testId)), new Path(testPath));
+
+ // load configuration file properties
+ Properties scaleProps = new Properties();
+ Properties testProps = new Properties();
+ try {
+ FileInputStream fis = new FileInputStream(sitePath);
+ try {
+ scaleProps.load(fis);
+ } finally {
+ fis.close();
+ }
+ fis = new FileInputStream(testPath);
+ try {
+ testProps.load(fis);
+ } finally {
+ fis.close();
+ }
+ } catch (Exception e) {
+ log.error("Error loading config file.", e);
+ }
+
+ ScaleTest test = (ScaleTest) Class.forName(String.format("org.apache.accumulo.test.scalability.%s", opts.testId)).newInstance();
+
+ test.init(scaleProps, testProps, opts.numTabletServers);
+
+ if (opts.action.equalsIgnoreCase("setup")) {
+ test.setup();
+ } else if (opts.action.equalsIgnoreCase("client")) {
+ InetAddress addr = InetAddress.getLocalHost();
+ String host = addr.getHostName();
+ fs.createNewFile(new Path("/accumulo-scale/clients/" + host));
+ test.client();
+ fs.copyFromLocalFile(new Path("/tmp/scale.out"), new Path("/accumulo-scale/results/" + host));
+ } else if (opts.action.equalsIgnoreCase("teardown")) {
+ test.teardown();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/scalability/ScaleTest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/scalability/ScaleTest.java b/core/src/main/java/org/apache/accumulo/testing/core/scalability/ScaleTest.java
new file mode 100644
index 0000000..a78f39e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/scalability/ScaleTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.scalability;
+
+import java.util.Properties;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.hadoop.io.Text;
+
+public abstract class ScaleTest {
+
+ private Connector conn;
+ private Properties scaleProps;
+ private Properties testProps;
+ private int numTabletServers;
+ private long startTime;
+
+ public void init(Properties scaleProps, Properties testProps, int numTabletServers) throws AccumuloException, AccumuloSecurityException {
+
+ this.scaleProps = scaleProps;
+ this.testProps = testProps;
+ this.numTabletServers = numTabletServers;
+
+ // get properties to create connector
+ String instanceName = this.scaleProps.getProperty("INSTANCE_NAME");
+ String zookeepers = this.scaleProps.getProperty("ZOOKEEPERS");
+ String user = this.scaleProps.getProperty("USER");
+ String password = this.scaleProps.getProperty("PASSWORD");
+ System.out.println(password);
+
+ conn = new ZooKeeperInstance(new ClientConfiguration().withInstance(instanceName).withZkHosts(zookeepers)).getConnector(user, new PasswordToken(password));
+ }
+
+ protected void startTimer() {
+ startTime = System.currentTimeMillis();
+ }
+
+ protected void stopTimer(long numEntries, long numBytes) {
+ long endTime = System.currentTimeMillis();
+ System.out.printf("ELAPSEDMS %d %d %d%n", endTime - startTime, numEntries, numBytes);
+ }
+
+ public abstract void setup();
+
+ public abstract void client();
+
+ public abstract void teardown();
+
+ public TreeSet<Text> calculateSplits() {
+ int numSplits = numTabletServers - 1;
+ long distance = (Long.MAX_VALUE / numTabletServers) + 1;
+ long split = distance;
+ TreeSet<Text> keys = new TreeSet<>();
+ for (int i = 0; i < numSplits; i++) {
+ keys.add(new Text(String.format("%016x", split)));
+ split += distance;
+ }
+ return keys;
+ }
+
+ public Connector getConnector() {
+ return conn;
+ }
+
+ public String getTestProperty(String key) {
+ return testProps.getProperty(key);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/DataWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/DataWriter.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/DataWriter.java
new file mode 100644
index 0000000..e7158e2
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/DataWriter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+
+public class DataWriter extends Stream<Void> {
+ private final BatchWriter writer;
+ private final RandomMutations mutations;
+
+ public DataWriter(BatchWriter writer, RandomMutations mutations) {
+ this.writer = writer;
+ this.mutations = mutations;
+ }
+
+ @Override
+ public Void next() {
+ try {
+ writer.addMutation(mutations.next());
+ } catch (MutationsRejectedException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public void finalize() {
+ try {
+ this.writer.close();
+ } catch (MutationsRejectedException e) {
+ System.err.println("Error closing batch writer.");
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/IntArgValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/IntArgValidator.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/IntArgValidator.java
new file mode 100644
index 0000000..5a5ad3e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/IntArgValidator.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.test.stress.random;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+
+import com.beust.jcommander.IValueValidator;
+import com.beust.jcommander.ParameterException;
+
+public class IntArgValidator implements IValueValidator<Integer> {
+
+ @Override
+ public void validate(String name, Integer value) throws ParameterException {
+ requireNonNull(value);
+ checkArgument(value > 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomByteArrays.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomByteArrays.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomByteArrays.java
new file mode 100644
index 0000000..405fabb
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomByteArrays.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+/**
+ * A stream that will create random byte arrays as it is looped over.
+ */
+public class RandomByteArrays extends Stream<byte[]> {
+ private final RandomWithinRange random_arrays;
+
+ public RandomByteArrays(RandomWithinRange random_arrays) {
+ this.random_arrays = random_arrays;
+ }
+
+ @Override
+ public byte[] next() {
+ return random_arrays.next_bytes();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomMutations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomMutations.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomMutations.java
new file mode 100644
index 0000000..db5da55
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomMutations.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.data.Mutation;
+
+public class RandomMutations extends Stream<Mutation> {
+ private final RandomByteArrays rows, column_families, column_qualifiers, values;
+ private final RandomWithinRange row_widths;
+ private final int max_cells_per_mutation;
+ private byte[] current_row;
+ private int cells_remaining_in_row;
+
+ public RandomMutations(RandomByteArrays rows, RandomByteArrays column_families, RandomByteArrays column_qualifiers, RandomByteArrays values,
+ RandomWithinRange row_widths, int max_cells_per_mutation) {
+ this.rows = rows;
+ this.column_families = column_families;
+ this.column_qualifiers = column_qualifiers;
+ this.values = values;
+ this.row_widths = row_widths;
+ this.max_cells_per_mutation = (max_cells_per_mutation > 0 ? max_cells_per_mutation : Integer.MAX_VALUE);
+
+ current_row = null;
+ cells_remaining_in_row = 0;
+ }
+
+ // TODO should we care about timestamps?
+ @Override
+ public Mutation next() {
+ if (cells_remaining_in_row == 0) {
+ current_row = rows.next();
+ cells_remaining_in_row = row_widths.next();
+ }
+ Mutation m = new Mutation(current_row);
+ final int cells = Math.min(cells_remaining_in_row, max_cells_per_mutation);
+ for (int i = 1; i <= cells; i++) {
+ m.put(column_families.next(), column_qualifiers.next(), values.next());
+ }
+ cells_remaining_in_row -= cells;
+ return m;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomWithinRange.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomWithinRange.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomWithinRange.java
new file mode 100644
index 0000000..06cea28
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomWithinRange.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.Random;
+
+/**
+ * Class that returns positive integers between some minimum and maximum.
+ *
+ */
+public class RandomWithinRange {
+ private final Random random;
+ private final int min, max;
+
+ public RandomWithinRange(int seed, int min, int max) {
+ this(new Random(seed), min, max);
+ }
+
+ public RandomWithinRange(Random random, int min, int max) {
+ checkArgument(min > 0, "Min must be positive.");
+ checkArgument(max >= min, "Max must be greater than or equal to min.");
+ this.random = random;
+ this.min = min;
+ this.max = max;
+ }
+
+ public int next() {
+ if (min == max) {
+ return min;
+ } else {
+ // we pick a random number that's between 0 and (max - min), then add
+ // min as an offset to get a random number that's [min, max)
+ return random.nextInt(max - min) + min;
+ }
+ }
+
+ public byte[] next_bytes() {
+ byte[] b = new byte[next()];
+ random.nextBytes(b);
+ return b;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java
new file mode 100644
index 0000000..3e8d5fd
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.collect.Lists;
+
+public class Scan {
+
+ public static void main(String[] args) throws Exception {
+ ScanOpts opts = new ScanOpts();
+ opts.parseArgs(Scan.class.getName(), args);
+
+ Connector connector = opts.getConnector();
+ Scanner scanner = connector.createScanner(opts.getTableName(), new Authorizations());
+
+ if (opts.isolate) {
+ scanner.enableIsolation();
+ }
+
+ Random tablet_index_generator = new Random(opts.scan_seed);
+
+ LoopControl scanning_condition = opts.continuous ? new ContinuousLoopControl() : new IterativeLoopControl(opts.scan_iterations);
+
+ while (scanning_condition.keepScanning()) {
+ Range range = pickRange(connector.tableOperations(), opts.getTableName(), tablet_index_generator);
+ scanner.setRange(range);
+ if (opts.batch_size > 0) {
+ scanner.setBatchSize(opts.batch_size);
+ }
+ try {
+ consume(scanner);
+ } catch (Exception e) {
+ System.err.println(String.format("Exception while scanning range %s. Check the state of Accumulo for errors.", range));
+ throw e;
+ }
+ }
+ }
+
+ public static void consume(Iterable<?> iterable) {
+ Iterator<?> itr = iterable.iterator();
+ while (itr.hasNext()) {
+ itr.next();
+ }
+ }
+
+ public static Range pickRange(TableOperations tops, String table, Random r) throws TableNotFoundException, AccumuloSecurityException, AccumuloException {
+ ArrayList<Text> splits = Lists.newArrayList(tops.listSplits(table));
+ if (splits.isEmpty()) {
+ return new Range();
+ } else {
+ int index = r.nextInt(splits.size());
+ Text endRow = splits.get(index);
+ Text startRow = index == 0 ? null : splits.get(index - 1);
+ return new Range(startRow, false, endRow, true);
+ }
+ }
+
+ /*
+ * These interfaces + implementations are used to determine how many times the scanner should look up a random tablet and scan it.
+ */
+ static interface LoopControl {
+ public boolean keepScanning();
+ }
+
+ // Does a finite number of iterations
+ static class IterativeLoopControl implements LoopControl {
+ private final int max;
+ private int current;
+
+ public IterativeLoopControl(int max) {
+ this.max = max;
+ this.current = 0;
+ }
+
+ @Override
+ public boolean keepScanning() {
+ if (current < max) {
+ ++current;
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ // Does an infinite number of iterations
+ static class ContinuousLoopControl implements LoopControl {
+ @Override
+ public boolean keepScanning() {
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/ScanOpts.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/ScanOpts.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/ScanOpts.java
new file mode 100644
index 0000000..e3f73f7
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/ScanOpts.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+
+import com.beust.jcommander.Parameter;
+
+class ScanOpts extends ClientOnDefaultTable {
+ @Parameter(names = "--isolate", description = "true to turn on scan isolation, false to turn off. default is false.")
+ boolean isolate = false;
+
+ @Parameter(names = "--num-iterations", description = "number of scan iterations")
+ int scan_iterations = 1024;
+
+ @Parameter(names = "--continuous", description = "continuously scan the table. note that this overrides --num-iterations")
+ boolean continuous;
+
+ @Parameter(names = "--scan-seed", description = "seed for randomly choosing tablets to scan")
+ int scan_seed = 1337;
+
+ @Parameter(names = "--scan-batch-size", description = "scanner batch size")
+ int batch_size = -1;
+
+ public ScanOpts() {
+ this(WriteOptions.DEFAULT_TABLE);
+ }
+
+ public ScanOpts(String table) {
+ super(table);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/Stream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Stream.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Stream.java
new file mode 100644
index 0000000..72b31e5
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Stream.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import java.util.Iterator;
+
+/**
+ * Base class to model an infinite stream of data. A stream implements an iterator whose {{@link #hasNext()} method will always return true.
+ *
+ */
+public abstract class Stream<T> implements Iterator<T> {
+
+ @Override
+ public final boolean hasNext() {
+ return true;
+ }
+
+ @Override
+ public abstract T next();
+
+ @Override
+ public final void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java
new file mode 100644
index 0000000..ea6f164
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+
+public class Write {
+
+ public static void main(String[] args) throws Exception {
+ WriteOptions opts = new WriteOptions();
+ BatchWriterOpts batch_writer_opts = new BatchWriterOpts();
+ opts.parseArgs(Write.class.getName(), args, batch_writer_opts);
+
+ opts.check();
+
+ Connector c = opts.getConnector();
+
+ if (opts.clear_table && c.tableOperations().exists(opts.getTableName())) {
+ try {
+ c.tableOperations().delete(opts.getTableName());
+ } catch (TableNotFoundException e) {
+ System.err.println("Couldn't delete the table because it doesn't exist any more.");
+ }
+ }
+
+ if (!c.tableOperations().exists(opts.getTableName())) {
+ try {
+ c.tableOperations().create(opts.getTableName());
+ } catch (TableExistsException e) {
+ System.err.println("Couldn't create table ourselves, but that's ok. Continuing.");
+ }
+ }
+
+ long writeDelay = opts.write_delay;
+ if (writeDelay < 0) {
+ writeDelay = 0;
+ }
+
+ DataWriter dw = new DataWriter(c.createBatchWriter(opts.getTableName(), batch_writer_opts.getBatchWriterConfig()), new RandomMutations(
+ // rows
+ new RandomByteArrays(new RandomWithinRange(opts.row_seed, opts.rowMin(), opts.rowMax())),
+ // cfs
+ new RandomByteArrays(new RandomWithinRange(opts.cf_seed, opts.cfMin(), opts.cfMax())),
+ // cqs
+ new RandomByteArrays(new RandomWithinRange(opts.cq_seed, opts.cqMin(), opts.cqMax())),
+ // vals
+ new RandomByteArrays(new RandomWithinRange(opts.value_seed, opts.valueMin(), opts.valueMax())),
+ // number of cells per row
+ new RandomWithinRange(opts.row_width_seed, opts.rowWidthMin(), opts.rowWidthMax()),
+ // max cells per mutation
+ opts.max_cells_per_mutation));
+
+ while (true) {
+ dw.next();
+ if (writeDelay > 0) {
+ Thread.sleep(writeDelay);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/WriteOptions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/WriteOptions.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/WriteOptions.java
new file mode 100644
index 0000000..f92a9eb
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/WriteOptions.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+
+import com.beust.jcommander.Parameter;
+
+class WriteOptions extends ClientOnDefaultTable {
+ static final String DEFAULT_TABLE = "stress_test";
+ static final int DEFAULT_MIN = 1, DEFAULT_MAX = 128, DEFAULT_SPREAD = DEFAULT_MAX - DEFAULT_MIN;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--min-row-size", description = "minimum row size")
+ Integer row_min;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--max-row-size", description = "maximum row size")
+ Integer row_max;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--min-cf-size", description = "minimum column family size")
+ Integer cf_min;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--max-cf-size", description = "maximum column family size")
+ Integer cf_max;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--min-cq-size", description = "minimum column qualifier size")
+ Integer cq_min;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--max-cq-size", description = "maximum column qualifier size")
+ Integer cq_max;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--min-value-size", description = "minimum value size")
+ Integer value_min;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--max-value-size", description = "maximum value size")
+ Integer value_max;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--min-row-width", description = "minimum row width")
+ Integer row_width_min;
+
+ @Parameter(validateValueWith = IntArgValidator.class, names = "--max-row-width", description = "maximum row width")
+ Integer row_width_max;
+
+ @Parameter(names = "--clear-table", description = "clears the table before ingesting")
+ boolean clear_table;
+
+ @Parameter(names = "--row-seed", description = "seed for generating rows")
+ int row_seed = 87;
+
+ @Parameter(names = "--cf-seed", description = "seed for generating column families")
+ int cf_seed = 7;
+
+ @Parameter(names = "--cq-seed", description = "seed for generating column qualifiers")
+ int cq_seed = 43;
+
+ @Parameter(names = "--value-seed", description = "seed for generating values")
+ int value_seed = 99;
+
+ @Parameter(names = "--row-width-seed", description = "seed for generating the number of cells within a row (a row's \"width\")")
+ int row_width_seed = 444;
+
+ @Parameter(names = "--max-cells-per-mutation", description = "maximum number of cells per mutation; non-positive value implies no limit")
+ int max_cells_per_mutation = -1;
+
+ @Parameter(names = "--write-delay", description = "milliseconds to wait between writes")
+ long write_delay = 0L;
+
+ public WriteOptions(String table) {
+ super(table);
+ }
+
+ public WriteOptions() {
+ this(DEFAULT_TABLE);
+ }
+
+ private static int minOrDefault(Integer ref) {
+ return ref == null ? DEFAULT_MIN : ref;
+ }
+
+ private static int calculateMax(Integer min_ref, Integer max_ref) {
+ if (max_ref == null) {
+ if (min_ref == null) {
+ return DEFAULT_MAX;
+ } else {
+ return min_ref + DEFAULT_SPREAD;
+ }
+ } else {
+ return max_ref;
+ }
+ }
+
+ public void check() {
+ checkPair("ROW", row_min, row_max);
+ checkPair("COLUMN FAMILY", cf_min, cf_max);
+ checkPair("COLUMN QUALIFIER", cq_min, cq_max);
+ checkPair("VALUE", value_min, value_max);
+ }
+
+ public void checkPair(String label, Integer min_ref, Integer max_ref) {
+ // we've already asserted that the numbers will either be
+ // 1) null
+ // 2) positive
+ // need to verify that they're coherent here
+
+ if (min_ref == null && max_ref != null) {
+ // we don't support just specifying a max yet
+ throw new IllegalArgumentException(String.format("[%s] Maximum value supplied, but no minimum. Must supply a minimum with a maximum value.", label));
+ } else if (min_ref != null && max_ref != null) {
+ // if a user supplied lower and upper bounds, we need to verify
+ // that min <= max
+ if (min_ref.compareTo(max_ref) > 0) {
+ throw new IllegalArgumentException(String.format("[%s] Min value (%d) is greater than max value (%d)", label, min_ref, max_ref));
+ }
+ }
+ }
+
+ public int rowMin() {
+ return minOrDefault(row_min);
+ }
+
+ public int rowMax() {
+ return calculateMax(row_min, row_max);
+ }
+
+ public int cfMin() {
+ return minOrDefault(cf_min);
+ }
+
+ public int cfMax() {
+ return calculateMax(cf_min, cf_max);
+ }
+
+ public int cqMin() {
+ return minOrDefault(cq_min);
+ }
+
+ public int cqMax() {
+ return calculateMax(cq_min, cq_max);
+ }
+
+ public int valueMin() {
+ return minOrDefault(value_min);
+ }
+
+ public int valueMax() {
+ return calculateMax(value_min, value_max);
+ }
+
+ public int rowWidthMin() {
+ return minOrDefault(row_width_min);
+ }
+
+ public int rowWidthMax() {
+ return calculateMax(row_width_min, row_width_max);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/package-info.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/package-info.java
new file mode 100644
index 0000000..fdbf72e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/package-info.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains utility classes designed to test Accumulo when large cells are being written. This is an attempt to observe the behavior Accumulo
+ * displays when compacting and reading these cells.
+ *
+ * There are two components to this package: {@link org.apache.accumulo.test.stress.random.Write} and {@link org.apache.accumulo.test.stress.random.Scan}.
+ *
+ * The {@link org.apache.accumulo.test.stress.random.Write} provides facilities for writing random sized cells. Users can configure minimum and maximum
+ * sized portions of a cell. The portions users can configure are the row, column family, column qualifier and value. Note that the sizes are uniformly
+ * distributed between the minimum and maximum values. See {@link org.apache.accumulo.test.stress.random.WriteOptions} for available options and default sizing
+ * information.
+ *
+ * The Scan provides users with the ability to query tables generated by the Write. It will pick a tablet at random and scan the entire range. The
+ * amount of times this process is done is user configurable. By default, it happens 1,024 times. Users can also specify whether or not the scan should be
+ * isolated or not.
+ *
+ * There is no shared state intended by either of these services. This allows multiple clients to be run in parallel, either on the same host or distributed
+ * across hosts.
+ */
+package org.apache.accumulo.test.stress.random;
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/.gitignore
----------------------------------------------------------------------
diff --git a/test/agitator/.gitignore b/test/agitator/.gitignore
new file mode 100644
index 0000000..3429b01
--- /dev/null
+++ b/test/agitator/.gitignore
@@ -0,0 +1,3 @@
+*~
+*.ini
+*.pyc
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/README.md
----------------------------------------------------------------------
diff --git a/test/agitator/README.md b/test/agitator/README.md
new file mode 100644
index 0000000..8abb74c
--- /dev/null
+++ b/test/agitator/README.md
@@ -0,0 +1,39 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Agitator: randomly kill processes
+===========================
+
+The agitator is used to randomly select processes for termination during
+system test.
+
+Configure the agitator using the example agitator.ini file provided.
+
+Create a list of hosts to be agitated:
+
+ $ cp ../../../conf/tservers hosts
+ $ echo master >> hosts
+ $ echo namenode >> hosts
+
+The agitator can be used to kill and restart any part of the accumulo
+ecosystem: zookeepers, namenode, datanodes, tablet servers and master.
+You can choose to agitate them all with "--all"
+
+ $ ./agitator.py --all --hosts=hosts --config=agitator.ini --log DEBUG
+
+You will need to be able to ssh, without passwords, to all your hosts as
+the user that can kill and start the services.
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/agitator.ini.example
----------------------------------------------------------------------
diff --git a/test/agitator/agitator.ini.example b/test/agitator/agitator.ini.example
new file mode 100644
index 0000000..3512561
--- /dev/null
+++ b/test/agitator/agitator.ini.example
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[DEFAULT]
+install=%(env.pwd)s/../../../..
+user=%(env.user)s
+
+[agitator]
+kill=kill -9
+ssh=ssh -q -A -o StrictHostKeyChecking=no
+sleep=300
+sleep.restart=30
+sleep.jitter=30
+
+[accumulo]
+home=%(install)s/accumulo
+tserver.kill.min=1
+tserver.kill.max=1
+tserver.frequency=0.8
+
+master.kill.min=1
+master.kill.max=1
+master.frequency=0.1
+
+gc.kill.min=1
+gc.kill.max=1
+gc.frequency=0.1
+
+[hadoop]
+home=%(install)s/hadoop
+bin=%(home)s/bin
+datanode.frequency=0.8
+datanode.kill.min=1
+datanode.kill.max=1
+namenode.frequency=0.05
+namenode.kill.min=1
+namenode.kill.max=1
+secondarynamenode.frequency=0.05
+secondarynamenode.kill.min=1
+secondarynamenode.kill.max=1
+
+[zookeeper]
+home=%(install)s/zookeeper
+frequency=0.05
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/agitator.py
----------------------------------------------------------------------
diff --git a/test/agitator/agitator.py b/test/agitator/agitator.py
new file mode 100755
index 0000000..db94546
--- /dev/null
+++ b/test/agitator/agitator.py
@@ -0,0 +1,241 @@
+#! /usr/bin/python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+import logging
+import ConfigParser
+
+# add the environment variables as default settings
+import os
+defaults=dict([('env.' + k, v) for k, v in os.environ.iteritems()])
+config = ConfigParser.ConfigParser(defaults)
+
+# things you can do to a particular kind of process
+class Proc:
+ program = 'Unknown'
+ _frequencyToKill = 1.0
+
+ def start(self, host):
+ pass
+
+ def find(self, host):
+ pass
+
+ def numberToKill(self):
+ return (1, 1)
+
+ def frequencyToKill(self):
+ return self._frequencyToKill
+
+ def user(self):
+ return config.get(self.program, 'user')
+
+ def kill(self, host, pid):
+ kill = config.get('agitator', 'kill').split()
+ code, stdout, stderr = self.runOn(host, kill + [pid])
+ if code != 0:
+ raise logging.warn("Unable to kill %d on %s (%s)", pid, host, stderr)
+
+ def runOn(self, host, cmd):
+ ssh = config.get('agitator', 'ssh').split()
+ return self.run(ssh + ["%s@%s" % (self.user(), host)] + cmd)
+
+ def run(self, cmd):
+ import subprocess
+ cmd = map(str, cmd)
+ logging.debug('Running %s', cmd)
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ if stdout.strip():
+ logging.debug("%s", stdout.strip())
+ if stderr.strip():
+ logging.error("%s", stderr.strip())
+ if p.returncode != 0:
+ logging.error("Problem running %s", ' '.join(cmd))
+ return p.returncode, stdout, stderr
+
+ def __repr__(self):
+ return self.program
+
+class Zookeeper(Proc):
+ program = 'zookeeper'
+ def __init__(self):
+ self._frequencyToKill = config.getfloat(self.program, 'frequency')
+
+ def start(self, host):
+ self.runOn(host, [config.get(self.program, 'home') + '/bin/zkServer.sh start'])
+
+ def find(self, host):
+ code, stdout, stderr = self.runOn(host, ['pgrep -f [Q]uorumPeerMain || true'])
+ return map(int, [line for line in stdout.split("\n") if line])
+
+class Hadoop(Proc):
+ section = 'hadoop'
+ def __init__(self, program):
+ self.program = program
+ self._frequencyToKill = config.getfloat(self.section, program + '.frequency')
+ self.minimumToKill = config.getint(self.section, program + '.kill.min')
+ self.maximumToKill = config.getint(self.section, program + '.kill.max')
+
+ def start(self, host):
+ binDir = config.get(self.section, 'bin')
+ self.runOn(host, ['nohup %s/hdfs %s < /dev/null >/dev/null 2>&1 &' %(binDir, self.program)])
+
+ def find(self, host):
+ code, stdout, stderr = self.runOn(host, ["pgrep -f 'proc[_]%s' || true" % (self.program,)])
+ return map(int, [line for line in stdout.split("\n") if line])
+
+ def numberToKill(self):
+ return (self.minimumToKill, self.maximumToKill)
+
+ def user(self):
+ return config.get(self.section, 'user')
+
+class Accumulo(Hadoop):
+ section = 'accumulo'
+ def start(self, host):
+ home = config.get(self.section, 'home')
+ self.runOn(host, ['nohup %s/bin/accumulo %s </dev/null >/dev/null 2>&1 & ' %(home, self.program)])
+
+ def find(self, host):
+ code, stdout, stderr = self.runOn(host, ["pgrep -f 'app[=]%s' || true" % self.program])
+ return map(int, [line for line in stdout.split("\n") if line])
+
+def fail(msg):
+ import sys
+ logging.critical(msg)
+ sys.exit(1)
+
+def jitter(n):
+ return random.random() * n - n / 2
+
+def sleep(n):
+ if n > 0:
+ logging.info("Sleeping %.2f", n)
+ import time
+ time.sleep(n)
+
+def agitate(hosts, procs):
+ starters = []
+
+ logging.info("Agitating %s on %d hosts" % (procs, len(hosts)))
+
+ section = 'agitator'
+
+ # repeatedly...
+ while True:
+ if starters:
+ # start up services that were previously killed
+ t = max(0, config.getfloat(section, 'sleep.restart') + jitter(config.getfloat(section, 'sleep.jitter')))
+ sleep(t)
+ for host, proc in starters:
+ logging.info('Starting %s on %s', proc, host)
+ proc.start(host)
+ starters = []
+
+ # wait some time
+ t = max(0, config.getfloat(section, 'sleep') + jitter(config.getfloat(section, 'sleep.jitter')))
+ sleep(t)
+
+ # for some processes
+ for p in procs:
+
+ # roll dice: should it be killed?
+ if random.random() < p.frequencyToKill():
+
+ # find them
+ from multiprocessing import Pool
+ def finder(host):
+ return host, p.find(host)
+ with Pool(5) as pool:
+ result = pool.map(finder, hosts)
+ candidates = {}
+ for host, pids in result:
+ if pids:
+ candidates[host] = pids
+
+ # how many?
+ minKill, maxKill = p.numberToKill()
+ count = min(random.randrange(minKill, maxKill + 1), len(candidates))
+
+ # pick the victims
+ doomedHosts = random.sample(candidates.keys(), count)
+
+ # kill them
+ logging.info("Killing %s on %s", p, doomedHosts)
+ for doomedHost in doomedHosts:
+ pids = candidates[doomedHost]
+ if not pids:
+ logging.error("Unable to kill any %s on %s: no processes of that type are running", p, doomedHost)
+ else:
+ pid = random.choice(pids)
+ logging.debug("Killing %s (%d) on %s", p, pid, doomedHost)
+ p.kill(doomedHost, pid)
+ # remember to restart them later
+ starters.append((doomedHost, p))
+
+def main():
+ import argparse
+ parser = argparse.ArgumentParser(description='Kill random processes')
+ parser.add_argument('--log', help='set the log level', default='INFO')
+ parser.add_argument('--namenodes', help='randomly kill namenodes', action="store_true")
+ parser.add_argument('--secondary', help='randomly kill secondary namenode', action="store_true")
+ parser.add_argument('--datanodes', help='randomly kill datanodes', action="store_true")
+ parser.add_argument('--tservers', help='randomly kill tservers', action="store_true")
+ parser.add_argument('--masters', help='randomly kill masters', action="store_true")
+ parser.add_argument('--zookeepers', help='randomly kill zookeepers', action="store_true")
+ parser.add_argument('--gc', help='randomly kill the file garbage collector', action="store_true")
+ parser.add_argument('--all',
+ help='kill any of the tservers, masters, datanodes, namenodes or zookeepers',
+ action='store_true')
+ parser.add_argument('--hosts', type=argparse.FileType('r'), required=True)
+ parser.add_argument('--config', type=argparse.FileType('r'), required=True)
+ args = parser.parse_args()
+
+ config.readfp(args.config)
+
+ level = getattr(logging, args.log.upper(), None)
+ if isinstance(level, int):
+ logging.basicConfig(level=level)
+
+ procs = []
+ def addIf(flag, proc):
+ if flag or args.all:
+ procs.append(proc)
+
+ addIf(args.namenodes, Hadoop('namenode'))
+ addIf(args.datanodes, Hadoop('datanode'))
+ addIf(args.secondary, Hadoop('secondarynamenode'))
+ addIf(args.tservers, Accumulo('tserver'))
+ addIf(args.masters, Accumulo('master'))
+ addIf(args.gc, Accumulo('gc'))
+ addIf(args.zookeepers, Zookeeper())
+ if len(procs) == 0:
+ fail("No processes to agitate!\n")
+
+ hosts = []
+ for line in args.hosts.readlines():
+ line = line.strip()
+ if line and line[0] != '#':
+ hosts.append(line)
+ if not hosts:
+ fail('No hosts to agitate!\n')
+
+ agitate(hosts, procs)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/hosts.example
----------------------------------------------------------------------
diff --git a/test/agitator/hosts.example b/test/agitator/hosts.example
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/test/agitator/hosts.example
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/README.md
----------------------------------------------------------------------
diff --git a/test/bench/README.md b/test/bench/README.md
new file mode 100644
index 0000000..0929bc3
--- /dev/null
+++ b/test/bench/README.md
@@ -0,0 +1,61 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+Benchmark Tests
+===============
+
+Running the Benchmarks
+----------------------
+
+Syntax for running run.py:
+
+> `$ ./run.py [-l -v <log_level> -s <run_speed> -u <user> -p <password> -i <instance>] [Benchmark1 ... BenchmarkN]`
+
+Specifying a specific benchmark or set of benchmarks runs only those, while
+not specifying any runs all benchmarks.
+
+`-l` lists the benchmarks that will be run
+`-v <run_speed>` can either be slow, medium or fast
+`-s <log_level>` is a number representing the verbosity of the debugging output: 10 is debug, 20 is info, 30 is warning, etc.
+`-u <user>` user to use when connecting with accumulo. If not set you will be prompted to input it.
+`-p <password>` password to use when connecting with accumulo. If not set you will be prompted to input it.
+`-z <zookeepers>` comma delimited lit of zookeeper host:port pairs to use when connecting with accumulo. If not set you will be prompted to input it.
+`-i <instance>` instance to use when connecting with accumulo. If not set you will be prompted to input it.
+
+The Benchmarks
+--------------
+
+Values in a 3-tuple are the slow,medium,fast speeds at which you can run the benchmarks.
+
+* CloudStone1: Test the speed at which we can check that accumulo is up and we can reach all the tservers. Lower is better.
+* CloudStone2: Ingest 10000,100000,1000000 rows of values 50 bytes on every tserver. Higher is better.
+* CloudStone3: Ingest 1000,5000,10000 rows of values 1024,8192,65535 bytes on every tserver. Higher is better.
+* CloudStone4 (TeraSort): Ingests 10000,10000000,10000000000 rows. Lower score is better.
+* CloudStone5: Creates 100,500,1000 tables named TestTableX and then deletes them. Lower is better.
+* CloudStone6: Creates a table with 400, 800, 1000 splits. Lower is better.
+
+Terasort
+--------
+
+The 4th Benchmark is Terasort. Run the benchmarks with speed 'slow' to do a full terasort.
+
+Misc
+----
+
+These benchmarks create tables in accumulo named 'test_ingest' and 'CloudIngestTest'. These tables are deleted
+at the end of the benchmarks. The benchmarks will also alter user auths while it runs. It is recommended that
+a benchmark user is created.
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone1/__init__.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone1/__init__.py b/test/bench/cloudstone1/__init__.py
new file mode 100755
index 0000000..09697dc
--- /dev/null
+++ b/test/bench/cloudstone1/__init__.py
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone1/cloudstone1.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone1/cloudstone1.py b/test/bench/cloudstone1/cloudstone1.py
new file mode 100755
index 0000000..309ef9c
--- /dev/null
+++ b/test/bench/cloudstone1/cloudstone1.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import time
+
+from lib import cloudshell
+from lib.Benchmark import Benchmark
+from lib.tservers import runAll
+from lib.path import accumulo
+
+class CloudStone1(Benchmark):
+
+ def shortDescription(self):
+ return 'Test the speed at which we can check that accumulo is up '\
+ 'and we can reach all the tservers. Lower is better.'
+
+ def runTest(self):
+ code, out, err = cloudshell.run(self.username, self.password, 'table accumulo.metadata\nscan\n')
+ self.assertEqual(code, 0, "Could not scan the metadata table. %s %s" % (out, err))
+ results = runAll('echo help | %s shell -u %s -p %s' %
+ (accumulo('bin', 'accumulo'), self.username, self.password))
+
+ def setSpeed(self, speed):
+ "We want to override this method but no speed can be set"
+
+def suite():
+ result = unittest.TestSuite([
+ CloudStone1(),
+ ])
+ return result
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone2/__init__.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone2/__init__.py b/test/bench/cloudstone2/__init__.py
new file mode 100755
index 0000000..09697dc
--- /dev/null
+++ b/test/bench/cloudstone2/__init__.py
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone2/cloudstone2.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone2/cloudstone2.py b/test/bench/cloudstone2/cloudstone2.py
new file mode 100755
index 0000000..996e5ef
--- /dev/null
+++ b/test/bench/cloudstone2/cloudstone2.py
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+from lib import cloudshell
+from lib.IngestBenchmark import IngestBenchmark
+
+class CloudStone2(IngestBenchmark):
+ "TestIngest one million small records on each tserver"
+
+ _size = 50
+ _count = 1000000
+
+ def size(self):
+ return self._size
+
+ def count(self):
+ return self._count
+
+ def setSpeed(self, speed):
+ if speed == "fast":
+ self._size = 50
+ self._count = 10000
+ elif speed == "medium":
+ self._size = 50
+ self._count = 100000
+ elif speed == "slow":
+ self._size = 50
+ self._count = 1000000
+
+def suite():
+ result = unittest.TestSuite([
+ CloudStone2(),
+ ])
+ return result
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone3/__init__.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone3/__init__.py b/test/bench/cloudstone3/__init__.py
new file mode 100755
index 0000000..09697dc
--- /dev/null
+++ b/test/bench/cloudstone3/__init__.py
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+