You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/01/03 20:55:51 UTC
[2/7] accumulo-testing git commit: ACCUMULO-4510 Adding Randomwalk
code from Accumulo
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java
new file mode 100644
index 0000000..eacd36b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Test deleting documents by using a compaction filter iterator
+ */
+public class CompactFilter extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String docTableName = (String) state.get("docTableName");
+ Random rand = (Random) state.get("rand");
+
+ String deleteChar = Integer.toHexString(rand.nextInt(16)) + "";
+ String regex = "^[0-9a-f][" + deleteChar + "].*";
+
+ ArrayList<IteratorSetting> documentFilters = new ArrayList<>();
+
+ IteratorSetting is = new IteratorSetting(21, "ii", RegExFilter.class);
+ RegExFilter.setRegexs(is, regex, null, null, null, false);
+ RegExFilter.setNegate(is, true);
+ documentFilters.add(is);
+
+ long t1 = System.currentTimeMillis();
+ env.getConnector().tableOperations().compact(docTableName, null, null, documentFilters, true, true);
+ long t2 = System.currentTimeMillis();
+ long t3 = t2 - t1;
+
+ ArrayList<IteratorSetting> indexFilters = new ArrayList<>();
+
+ is = new IteratorSetting(21, RegExFilter.class);
+ RegExFilter.setRegexs(is, null, null, regex, null, false);
+ RegExFilter.setNegate(is, true);
+ indexFilters.add(is);
+
+ t1 = System.currentTimeMillis();
+ env.getConnector().tableOperations().compact(indexTableName, null, null, indexFilters, true, true);
+ t2 = System.currentTimeMillis();
+
+ log.debug("Filtered documents using compaction iterators " + regex + " " + (t3) + " " + (t2 - t1));
+
+ BatchScanner bscanner = env.getConnector().createBatchScanner(docTableName, new Authorizations(), 10);
+
+ List<Range> ranges = new ArrayList<>();
+ for (int i = 0; i < 16; i++) {
+ ranges.add(Range.prefix(new Text(Integer.toHexString(i) + "" + deleteChar)));
+ }
+
+ bscanner.setRanges(ranges);
+ Iterator<Entry<Key,Value>> iter = bscanner.iterator();
+
+ if (iter.hasNext()) {
+ throw new Exception("Saw unexpected document " + iter.next().getKey());
+ }
+
+ bscanner.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java
new file mode 100644
index 0000000..e2c8bea
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Delete extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+ Random rand = (Random) state.get("rand");
+
+ Entry<Key,Value> entry = Search.findRandomDocument(state, env, dataTableName, rand);
+ if (entry == null)
+ return;
+
+ String docID = entry.getKey().getRow().toString();
+ String doc = entry.getValue().toString();
+
+ Insert.unindexDocument(env.getMultiTableBatchWriter().getBatchWriter(indexTableName), doc, docID, numPartitions);
+
+ Mutation m = new Mutation(docID);
+ m.putDelete("doc", "");
+
+ env.getMultiTableBatchWriter().getBatchWriter(dataTableName).addMutation(m);
+
+ log.debug("Deleted document " + docID);
+
+ env.getMultiTableBatchWriter().flush();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java
new file mode 100644
index 0000000..2b790bd
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchDeleter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+//a test created to test the batch deleter
+public class DeleteSomeDocs extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ // delete documents that where the document id matches a given pattern from doc and index table
+ // using the batch deleter
+
+ Random rand = (Random) state.get("rand");
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+
+ ArrayList<String> patterns = new ArrayList<>();
+
+ for (Object key : props.keySet())
+ if (key instanceof String && ((String) key).startsWith("pattern"))
+ patterns.add(props.getProperty((String) key));
+
+ String pattern = patterns.get(rand.nextInt(patterns.size()));
+ BatchWriterConfig bwc = new BatchWriterConfig();
+ BatchDeleter ibd = env.getConnector().createBatchDeleter(indexTableName, Authorizations.EMPTY, 8, bwc);
+ ibd.setRanges(Collections.singletonList(new Range()));
+
+ IteratorSetting iterSettings = new IteratorSetting(100, RegExFilter.class);
+ RegExFilter.setRegexs(iterSettings, null, null, pattern, null, false);
+
+ ibd.addScanIterator(iterSettings);
+
+ ibd.delete();
+
+ ibd.close();
+
+ BatchDeleter dbd = env.getConnector().createBatchDeleter(dataTableName, Authorizations.EMPTY, 8, bwc);
+ dbd.setRanges(Collections.singletonList(new Range()));
+
+ iterSettings = new IteratorSetting(100, RegExFilter.class);
+ RegExFilter.setRegexs(iterSettings, pattern, null, null, null, false);
+
+ dbd.addScanIterator(iterSettings);
+
+ dbd.delete();
+
+ dbd.close();
+
+ log.debug("Deleted documents w/ id matching '" + pattern + "'");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java
new file mode 100644
index 0000000..544c35e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Delete all documents containing a particular word.
+ *
+ */
+
+public class DeleteWord extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String docTableName = (String) state.get("docTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+ Random rand = (Random) state.get("rand");
+
+ String wordToDelete = Insert.generateRandomWord(rand);
+
+ // use index to find all documents containing word
+ Scanner scanner = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(new Text(wordToDelete));
+
+ ArrayList<Range> documentsToDelete = new ArrayList<>();
+
+ for (Entry<Key,Value> entry : scanner)
+ documentsToDelete.add(new Range(entry.getKey().getColumnQualifier()));
+
+ if (documentsToDelete.size() > 0) {
+ // use a batch scanner to fetch all documents
+ BatchScanner bscanner = env.getConnector().createBatchScanner(docTableName, Authorizations.EMPTY, 8);
+ bscanner.setRanges(documentsToDelete);
+
+ BatchWriter ibw = env.getMultiTableBatchWriter().getBatchWriter(indexTableName);
+ BatchWriter dbw = env.getMultiTableBatchWriter().getBatchWriter(docTableName);
+
+ int count = 0;
+
+ for (Entry<Key,Value> entry : bscanner) {
+ String docID = entry.getKey().getRow().toString();
+ String doc = entry.getValue().toString();
+
+ Insert.unindexDocument(ibw, doc, docID, numPartitions);
+
+ Mutation m = new Mutation(docID);
+ m.putDelete("doc", "");
+
+ dbw.addMutation(m);
+ count++;
+ }
+
+ bscanner.close();
+
+ env.getMultiTableBatchWriter().flush();
+
+ if (count != documentsToDelete.size()) {
+ throw new Exception("Batch scanner did not return expected number of docs " + count + " " + documentsToDelete.size());
+ }
+ }
+
+ log.debug("Deleted " + documentsToDelete.size() + " documents containing " + wordToDelete);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
new file mode 100644
index 0000000..d52198b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class ExportIndex extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ String indexTableName = (String) state.get("indexTableName");
+ String tmpIndexTableName = indexTableName + "_tmp";
+
+ String exportDir = "/tmp/shard_export/" + indexTableName;
+ String copyDir = "/tmp/shard_export/" + tmpIndexTableName;
+
+ FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+
+ fs.delete(new Path("/tmp/shard_export/" + indexTableName), true);
+ fs.delete(new Path("/tmp/shard_export/" + tmpIndexTableName), true);
+
+ // disable spits, so that splits can be compared later w/o worrying one table splitting and the other not
+ env.getConnector().tableOperations().setProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "20G");
+
+ long t1 = System.currentTimeMillis();
+
+ env.getConnector().tableOperations().flush(indexTableName, null, null, true);
+ env.getConnector().tableOperations().offline(indexTableName);
+
+ long t2 = System.currentTimeMillis();
+
+ env.getConnector().tableOperations().exportTable(indexTableName, exportDir);
+
+ long t3 = System.currentTimeMillis();
+
+ // copy files
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(new Path(exportDir, "distcp.txt")), UTF_8));
+ String file = null;
+ while ((file = reader.readLine()) != null) {
+ Path src = new Path(file);
+ Path dest = new Path(new Path(copyDir), src.getName());
+ FileUtil.copy(fs, src, fs, dest, false, true, CachedConfiguration.getInstance());
+ }
+
+ reader.close();
+
+ long t4 = System.currentTimeMillis();
+
+ env.getConnector().tableOperations().online(indexTableName);
+ env.getConnector().tableOperations().importTable(tmpIndexTableName, copyDir);
+
+ long t5 = System.currentTimeMillis();
+
+ fs.delete(new Path(exportDir), true);
+ fs.delete(new Path(copyDir), true);
+
+ HashSet<Text> splits1 = new HashSet<>(env.getConnector().tableOperations().listSplits(indexTableName));
+ HashSet<Text> splits2 = new HashSet<>(env.getConnector().tableOperations().listSplits(tmpIndexTableName));
+
+ if (!splits1.equals(splits2))
+ throw new Exception("Splits not equals " + indexTableName + " " + tmpIndexTableName);
+
+ HashMap<String,String> props1 = new HashMap<>();
+ for (Entry<String,String> entry : env.getConnector().tableOperations().getProperties(indexTableName))
+ props1.put(entry.getKey(), entry.getValue());
+
+ HashMap<String,String> props2 = new HashMap<>();
+ for (Entry<String,String> entry : env.getConnector().tableOperations().getProperties(tmpIndexTableName))
+ props2.put(entry.getKey(), entry.getValue());
+
+ if (!props1.equals(props2))
+ throw new Exception("Props not equals " + indexTableName + " " + tmpIndexTableName);
+
+ // unset the split threshold
+ env.getConnector().tableOperations().removeProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey());
+ env.getConnector().tableOperations().removeProperty(tmpIndexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey());
+
+ log.debug("Imported " + tmpIndexTableName + " from " + indexTableName + " flush: " + (t2 - t1) + "ms export: " + (t3 - t2) + "ms copy:" + (t4 - t3)
+ + "ms import:" + (t5 - t4) + "ms");
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java
new file mode 100644
index 0000000..e6ac574
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Flush extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+ Random rand = (Random) state.get("rand");
+
+ String table;
+
+ if (rand.nextDouble() < .5)
+ table = indexTableName;
+ else
+ table = dataTableName;
+
+ env.getConnector().tableOperations().flush(table, null, null, true);
+ log.debug("Flushed " + table);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java
new file mode 100644
index 0000000..5892626
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.IntersectingIterator;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Grep extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ // pick a few randoms words... grep for those words and search the index
+ // ensure both return the same set of documents
+
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+ Random rand = (Random) state.get("rand");
+
+ Text words[] = new Text[rand.nextInt(4) + 2];
+
+ for (int i = 0; i < words.length; i++) {
+ words[i] = new Text(Insert.generateRandomWord(rand));
+ }
+
+ BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 16);
+ IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class.getName());
+ IntersectingIterator.setColumnFamilies(ii, words);
+ bs.addScanIterator(ii);
+ bs.setRanges(Collections.singleton(new Range()));
+
+ HashSet<Text> documentsFoundInIndex = new HashSet<>();
+
+ for (Entry<Key,Value> entry2 : bs) {
+ documentsFoundInIndex.add(entry2.getKey().getColumnQualifier());
+ }
+
+ bs.close();
+
+ bs = env.getConnector().createBatchScanner(dataTableName, Authorizations.EMPTY, 16);
+
+ for (int i = 0; i < words.length; i++) {
+ IteratorSetting more = new IteratorSetting(20 + i, "ii" + i, RegExFilter.class);
+ RegExFilter.setRegexs(more, null, null, null, "(^|(.*\\s))" + words[i] + "($|(\\s.*))", false);
+ bs.addScanIterator(more);
+ }
+
+ bs.setRanges(Collections.singleton(new Range()));
+
+ HashSet<Text> documentsFoundByGrep = new HashSet<>();
+
+ for (Entry<Key,Value> entry2 : bs) {
+ documentsFoundByGrep.add(entry2.getKey().getRow());
+ }
+
+ bs.close();
+
+ if (!documentsFoundInIndex.equals(documentsFoundByGrep)) {
+ throw new Exception("Set of documents found not equal for words " + Arrays.asList(words).toString() + " " + documentsFoundInIndex + " "
+ + documentsFoundByGrep);
+ }
+
+ log.debug("Grep and index agree " + Arrays.asList(words).toString() + " " + documentsFoundInIndex.size());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java
new file mode 100644
index 0000000..1b15323
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Insert extends Test {
+
+ static final int NUM_WORDS = 100000;
+ static final int MIN_WORDS_PER_DOC = 10;
+ static final int MAX_WORDS_PER_DOC = 3000;
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+ Random rand = (Random) state.get("rand");
+ long nextDocID = (Long) state.get("nextDocID");
+
+ BatchWriter dataWriter = env.getMultiTableBatchWriter().getBatchWriter(dataTableName);
+ BatchWriter indexWriter = env.getMultiTableBatchWriter().getBatchWriter(indexTableName);
+
+ String docID = insertRandomDocument(nextDocID++, dataWriter, indexWriter, indexTableName, dataTableName, numPartitions, rand);
+
+ log.debug("Inserted document " + docID);
+
+ state.set("nextDocID", Long.valueOf(nextDocID));
+ }
+
+ static String insertRandomDocument(long did, BatchWriter dataWriter, BatchWriter indexWriter, String indexTableName, String dataTableName, int numPartitions,
+ Random rand) throws TableNotFoundException, Exception, AccumuloException, AccumuloSecurityException {
+ String doc = createDocument(rand);
+
+ String docID = new StringBuilder(String.format("%016x", did)).reverse().toString();
+
+ saveDocument(dataWriter, docID, doc);
+ indexDocument(indexWriter, doc, docID, numPartitions);
+
+ return docID;
+ }
+
+ static void saveDocument(BatchWriter bw, String docID, String doc) throws Exception {
+
+ Mutation m = new Mutation(docID);
+ m.put("doc", "", doc);
+
+ bw.addMutation(m);
+ }
+
+ static String createDocument(Random rand) {
+ StringBuilder sb = new StringBuilder();
+
+ int numWords = rand.nextInt(MAX_WORDS_PER_DOC - MIN_WORDS_PER_DOC) + MIN_WORDS_PER_DOC;
+
+ for (int i = 0; i < numWords; i++) {
+ String word = generateRandomWord(rand);
+
+ if (i > 0)
+ sb.append(" ");
+
+ sb.append(word);
+ }
+
+ return sb.toString();
+ }
+
+ static String generateRandomWord(Random rand) {
+ return Integer.toString(rand.nextInt(NUM_WORDS), Character.MAX_RADIX);
+ }
+
+ static String genPartition(int partition) {
+ return String.format("%06x", Math.abs(partition));
+ }
+
+ static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception {
+ indexDocument(bw, doc, docId, numPartitions, false);
+ }
+
+ static void unindexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception {
+ indexDocument(bw, doc, docId, numPartitions, true);
+ }
+
+ static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions, boolean delete) throws Exception {
+
+ String[] tokens = doc.split("\\W+");
+
+ String partition = genPartition(doc.hashCode() % numPartitions);
+
+ Mutation m = new Mutation(partition);
+
+ HashSet<String> tokensSeen = new HashSet<>();
+
+ for (String token : tokens) {
+ token = token.toLowerCase();
+
+ if (!tokensSeen.contains(token)) {
+ tokensSeen.add(token);
+ if (delete)
+ m.putDelete(token, docId);
+ else
+ m.put(token, docId, new Value(new byte[0]));
+ }
+ }
+
+ if (m.size() > 0)
+ bw.addMutation(m);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java
new file mode 100644
index 0000000..106ab3b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Collection;
+import java.util.Properties;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Merge extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+
+ Collection<Text> splits = env.getConnector().tableOperations().listSplits(indexTableName);
+ SortedSet<Text> splitSet = new TreeSet<>(splits);
+ log.debug("merging " + indexTableName);
+ env.getConnector().tableOperations().merge(indexTableName, null, null);
+ org.apache.accumulo.core.util.Merge merge = new org.apache.accumulo.core.util.Merge();
+ merge.mergomatic(env.getConnector(), indexTableName, null, null, 256 * 1024 * 1024, true);
+ splits = env.getConnector().tableOperations().listSplits(indexTableName);
+ if (splits.size() > splitSet.size()) {
+ // throw an excpetion so that test will die an no further changes to table will occur...
+ // this way table is left as is for debugging.
+ throw new Exception("There are more tablets after a merge: " + splits.size() + " was " + splitSet.size());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java
new file mode 100644
index 0000000..ac0c872
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Reindex extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String tmpIndexTableName = indexTableName + "_tmp";
+ String docTableName = (String) state.get("docTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+
+ Random rand = (Random) state.get("rand");
+
+ ShardFixture.createIndexTable(this.log, state, env, "_tmp", rand);
+
+ Scanner scanner = env.getConnector().createScanner(docTableName, Authorizations.EMPTY);
+ BatchWriter tbw = env.getConnector().createBatchWriter(tmpIndexTableName, new BatchWriterConfig());
+
+ int count = 0;
+
+ for (Entry<Key,Value> entry : scanner) {
+ String docID = entry.getKey().getRow().toString();
+ String doc = entry.getValue().toString();
+
+ Insert.indexDocument(tbw, doc, docID, numPartitions);
+
+ count++;
+ }
+
+ tbw.close();
+
+ log.debug("Reindexed " + count + " documents into " + tmpIndexTableName);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java
new file mode 100644
index 0000000..c07397d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.IntersectingIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Search extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ String dataTableName = (String) state.get("docTableName");
+
+ Random rand = (Random) state.get("rand");
+
+ Entry<Key,Value> entry = findRandomDocument(state, env, dataTableName, rand);
+ if (entry == null)
+ return;
+
+ Text docID = entry.getKey().getRow();
+ String doc = entry.getValue().toString();
+
+ String[] tokens = doc.split("\\W+");
+ int numSearchTerms = rand.nextInt(6);
+ if (numSearchTerms < 2)
+ numSearchTerms = 2;
+
+ HashSet<String> searchTerms = new HashSet<>();
+ while (searchTerms.size() < numSearchTerms)
+ searchTerms.add(tokens[rand.nextInt(tokens.length)]);
+
+ Text columns[] = new Text[searchTerms.size()];
+ int index = 0;
+ for (String term : searchTerms) {
+ columns[index++] = new Text(term);
+ }
+
+ log.debug("Looking up terms " + searchTerms + " expect to find " + docID);
+
+ BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 10);
+ IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class);
+ IntersectingIterator.setColumnFamilies(ii, columns);
+ bs.addScanIterator(ii);
+ bs.setRanges(Collections.singleton(new Range()));
+
+ boolean sawDocID = false;
+
+ for (Entry<Key,Value> entry2 : bs) {
+ if (entry2.getKey().getColumnQualifier().equals(docID)) {
+ sawDocID = true;
+ break;
+ }
+ }
+
+ bs.close();
+
+ if (!sawDocID)
+ throw new Exception("Did not see doc " + docID + " in index. terms:" + searchTerms + " " + indexTableName + " " + dataTableName);
+ }
+
+ static Entry<Key,Value> findRandomDocument(State state, Environment env, String dataTableName, Random rand) throws Exception {
+ Scanner scanner = env.getConnector().createScanner(dataTableName, Authorizations.EMPTY);
+ scanner.setBatchSize(1);
+ scanner.setRange(new Range(Integer.toString(rand.nextInt(0xfffffff), 16), null));
+
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+ if (!iter.hasNext())
+ return null;
+
+ return iter.next();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java
new file mode 100644
index 0000000..e20f64d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.net.InetAddress;
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.Fixture;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class ShardFixture extends Fixture {
+
+ static SortedSet<Text> genSplits(long max, int numTablets, String format) {
+
+ int numSplits = numTablets - 1;
+ long distance = max / numTablets;
+ long split = distance;
+
+ TreeSet<Text> splits = new TreeSet<>();
+
+ for (int i = 0; i < numSplits; i++) {
+ splits.add(new Text(String.format(format, split)));
+ split += distance;
+ }
+
+ return splits;
+ }
+
+ static void createIndexTable(Logger log, State state, Environment env, String suffix, Random rand) throws Exception {
+ Connector conn = env.getConnector();
+ String name = (String) state.get("indexTableName") + suffix;
+ int numPartitions = (Integer) state.get("numPartitions");
+ boolean enableCache = (Boolean) state.get("cacheIndex");
+ conn.tableOperations().create(name);
+
+ String tableId = conn.tableOperations().tableIdMap().get(name);
+ log.info("Created index table " + name + "(id:" + tableId + ")");
+
+ SortedSet<Text> splits = genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x");
+ conn.tableOperations().addSplits(name, splits);
+
+ log.info("Added " + splits.size() + " splits to " + name);
+
+ if (enableCache) {
+ conn.tableOperations().setProperty(name, Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
+ conn.tableOperations().setProperty(name, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
+
+ log.info("Enabled caching for table " + name);
+ }
+ }
+
+ @Override
+ public void setUp(State state, Environment env) throws Exception {
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+ String pid = env.getPid();
+
+ Random rand = new Random();
+
+ int numPartitions = rand.nextInt(90) + 10;
+
+ state.set("indexTableName", String.format("ST_index_%s_%s_%d", hostname, pid, System.currentTimeMillis()));
+ state.set("docTableName", String.format("ST_docs_%s_%s_%d", hostname, pid, System.currentTimeMillis()));
+ state.set("numPartitions", Integer.valueOf(numPartitions));
+ state.set("cacheIndex", rand.nextDouble() < .5);
+ state.set("rand", rand);
+ state.set("nextDocID", Long.valueOf(0));
+
+ Connector conn = env.getConnector();
+
+ createIndexTable(this.log, state, env, "", rand);
+
+ String docTableName = (String) state.get("docTableName");
+ conn.tableOperations().create(docTableName);
+
+ String tableId = conn.tableOperations().tableIdMap().get(docTableName);
+ log.info("Created doc table " + docTableName + " (id:" + tableId + ")");
+
+ SortedSet<Text> splits = genSplits(0xff, rand.nextInt(32) + 1, "%02x");
+ conn.tableOperations().addSplits(docTableName, splits);
+
+ log.info("Added " + splits.size() + " splits to " + docTableName);
+
+ if (rand.nextDouble() < .5) {
+ conn.tableOperations().setProperty((String) state.get("docTableName"), Property.TABLE_BLOOM_ENABLED.getKey(), "true");
+ log.info("Enabled bloom filters for table " + (String) state.get("docTableName"));
+ }
+ }
+
+ @Override
+ public void tearDown(State state, Environment env) throws Exception {
+ // We have resources we need to clean up
+ if (env.isMultiTableBatchWriterInitialized()) {
+ MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ log.error("Ignoring mutations that weren't flushed", e);
+ }
+
+ // Reset the MTBW on the state to null
+ env.resetMultiTableBatchWriter();
+ }
+
+ Connector conn = env.getConnector();
+
+ log.info("Deleting index and doc tables");
+
+ conn.tableOperations().delete((String) state.get("indexTableName"));
+ conn.tableOperations().delete((String) state.get("docTableName"));
+
+ log.debug("Exiting shard test");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java
new file mode 100644
index 0000000..04c96f4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Collection;
+
+import org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat;
+import org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SortTool extends Configured implements Tool {
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+ private String outputDir;
+ private String seqFile;
+ private String splitFile;
+ private Collection<Text> splits;
+
+ public SortTool(String seqFile, String outputDir, String splitFile, Collection<Text> splits) {
+ this.outputDir = outputDir;
+ this.seqFile = seqFile;
+ this.splitFile = splitFile;
+ this.splits = splits;
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Job job = Job.getInstance(getConf(), this.getClass().getSimpleName());
+ job.setJarByClass(this.getClass());
+
+ if (job.getJar() == null) {
+ log.error("M/R requires a jar file! Run mvn package.");
+ return 1;
+ }
+
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ SequenceFileInputFormat.setInputPaths(job, seqFile);
+
+ job.setPartitionerClass(KeyRangePartitioner.class);
+ KeyRangePartitioner.setSplitFile(job, splitFile);
+
+ job.setMapOutputKeyClass(Key.class);
+ job.setMapOutputValueClass(Value.class);
+
+ job.setNumReduceTasks(splits.size() + 1);
+
+ job.setOutputFormatClass(AccumuloFileOutputFormat.class);
+ AccumuloFileOutputFormat.setOutputPath(job, new Path(outputDir));
+
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java
new file mode 100644
index 0000000..bef5104
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Properties;
+import java.util.Random;
+import java.util.SortedSet;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Split extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+ String indexTableName = (String) state.get("indexTableName");
+ int numPartitions = (Integer) state.get("numPartitions");
+ Random rand = (Random) state.get("rand");
+
+ SortedSet<Text> splitSet = ShardFixture.genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x");
+ log.debug("adding splits " + indexTableName);
+ env.getConnector().tableOperations().addSplits(indexTableName, splitSet);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java
new file mode 100644
index 0000000..caba1d7
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class VerifyIndex extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {
+
+ String indexTableName = (String) state.get("indexTableName");
+ String tmpIndexTableName = indexTableName + "_tmp";
+
+ // scan new and old index and verify identical
+ Scanner indexScanner1 = env.getConnector().createScanner(tmpIndexTableName, Authorizations.EMPTY);
+ Scanner indexScanner2 = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY);
+
+ Iterator<Entry<Key,Value>> iter = indexScanner2.iterator();
+
+ int count = 0;
+
+ for (Entry<Key,Value> entry : indexScanner1) {
+ if (!iter.hasNext())
+ throw new Exception("index rebuild mismatch " + entry.getKey() + " " + indexTableName);
+
+ Key key1 = entry.getKey();
+ Key key2 = iter.next().getKey();
+
+ if (!key1.equals(key2, PartialKey.ROW_COLFAM_COLQUAL))
+ throw new Exception("index rebuild mismatch " + key1 + " " + key2 + " " + indexTableName + " " + tmpIndexTableName);
+ count++;
+ if (count % 1000 == 0)
+ makingProgress();
+ }
+
+ if (iter.hasNext())
+ throw new Exception("index rebuild mismatch " + iter.next().getKey() + " " + tmpIndexTableName);
+
+ log.debug("Verified " + count + " index entries ");
+
+ env.getConnector().tableOperations().delete(indexTableName);
+ env.getConnector().tableOperations().rename(tmpIndexTableName, indexTableName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java
new file mode 100644
index 0000000..df1df59
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CreateTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java
new file mode 100644
index 0000000..f7226cb
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class DeleteTable extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java
new file mode 100644
index 0000000..5681402
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Ingest extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java
new file mode 100644
index 0000000..c677cf9
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Scan extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java
new file mode 100644
index 0000000..95acf4f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Verify extends Test {
+
+ @Override
+ public void visit(State state, Environment env, Properties props) throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/module.xsd
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/module.xsd b/core/src/main/resources/randomwalk/module.xsd
new file mode 100644
index 0000000..bcdaaae
--- /dev/null
+++ b/core/src/main/resources/randomwalk/module.xsd
@@ -0,0 +1,69 @@
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+ <xsd:element name="module" type="ModuleType"/>
+
+ <xsd:complexType name="ModuleType">
+ <xsd:sequence>
+ <xsd:element name="package" type="PrefixType" minOccurs="0" maxOccurs="unbounded"/>
+ <xsd:element name="fixture" type="InitType" minOccurs="0" maxOccurs="1"/>
+ <xsd:element name="init" type="InitType"/>
+ <xsd:element name="node" type="NodeType" minOccurs="1" maxOccurs="unbounded"/>
+ </xsd:sequence>
+ </xsd:complexType>
+
+ <xsd:complexType name="PrefixType">
+ <xsd:attribute name="prefix" type="xsd:string"/>
+ <xsd:attribute name="value" type="xsd:string"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="InitType">
+ <xsd:attribute name="id" type="xsd:string"/>
+ <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
+ <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
+ <xsd:attribute name="teardown" type="xsd:boolean"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="NodeType">
+ <xsd:sequence>
+ <xsd:element name="alias" type="AliasType" minOccurs="0" maxOccurs="unbounded"/>
+ <xsd:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
+ <xsd:element name="edge" type="EdgeType" minOccurs="1" maxOccurs="unbounded"/>
+ </xsd:sequence>
+ <xsd:attribute name="id" type="xsd:string"/>
+ <xsd:attribute name="src" type="xsd:string"/>
+ <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
+ <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
+ <xsd:attribute name="teardown" type="xsd:boolean"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="EdgeType">
+ <xsd:attribute name="id" type="xsd:string"/>
+ <xsd:attribute name="weight" type="xsd:positiveInteger"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="AliasType">
+ <xsd:attribute name="name" type="xsd:string"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="PropertyType">
+ <xsd:attribute name="key" type="xsd:string"/>
+ <xsd:attribute name="value" type="xsd:string"/>
+ </xsd:complexType>
+
+</xsd:schema>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/All.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/All.xml b/core/src/main/resources/randomwalk/modules/All.xml
new file mode 100644
index 0000000..be5c815
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/All.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<init id="dummy.ToAll"/>
+
+<node id="Image.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="Image.xml" weight="1"/>
+ <edge id="Sequential.xml" weight="1"/>
+ <edge id="MultiTable.xml" weight="1"/>
+ <edge id="Shard.xml" weight="1"/>
+ <edge id="Concurrent.xml" weight="1"/>
+ <edge id="Conditional.xml" weight="1"/>
+ <edge id="Security.xml" weight="1"/>
+ <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="MultiTable.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Shard.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Concurrent.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Conditional.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Security.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Bulk.xml">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Bulk.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Bulk.xml b/core/src/main/resources/randomwalk/modules/Bulk.xml
new file mode 100644
index 0000000..35e2a67
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Bulk.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="bulk" value="org.apache.accumulo.test.randomwalk.bulk"/>
+
+<init id="bulk.Setup"/>
+
+<node id="bulk.Setup">
+ <edge id="bulk.BulkPlusOne" weight="1"/>
+</node>
+
+<node id="bulk.BulkPlusOne">
+ <edge id="bulk.Merge" weight="10"/>
+ <edge id="bulk.Split" weight="10"/>
+ <edge id="bulk.Compact" weight="10"/>
+ <edge id="bulk.ConsistencyCheck" weight="25"/>
+ <edge id="bulk.BulkMinusOne" weight="10"/>
+</node>
+
+<node id="bulk.BulkMinusOne">
+ <edge id="bulk.BulkPlusOne" weight="200"/>
+ <edge id="bulk.Verify" weight="1"/>
+</node>
+
+<node id="bulk.Merge">
+ <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.Split">
+ <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.Compact">
+ <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.ConsistencyCheck">
+ <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.Verify">
+ <edge id="END" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Concurrent.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Concurrent.xml b/core/src/main/resources/randomwalk/modules/Concurrent.xml
new file mode 100644
index 0000000..36ea53c
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Concurrent.xml
@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="ct" value="org.apache.accumulo.test.randomwalk.concurrent"/>
+
+<fixture id="ct.ConcurrentFixture"/>
+
+<init id="ct.Setup"/>
+
+<node id="dummy.ToAll">
+ <edge id="ct.CreateTable" weight="1000"/>
+ <edge id="ct.BatchWrite" weight="1000"/>
+ <edge id="ct.BatchScan" weight="1000"/>
+ <edge id="ct.CloneTable" weight="1000"/>
+ <edge id="ct.DeleteTable" weight="1000"/>
+ <edge id="ct.RenameTable" weight="1000"/>
+ <edge id="ct.ScanTable" weight="500"/>
+ <edge id="ct.Replication" weight="500"/>
+ <edge id="ct.IsolatedScan" weight="500"/>
+ <edge id="ct.AddSplits" weight="1000"/>
+ <edge id="ct.ListSplits" weight="1000"/>
+ <edge id="ct.OfflineTable" weight="1000"/>
+ <edge id="ct.Merge" weight="1000"/>
+ <edge id="ct.Compact" weight="1000"/>
+ <edge id="ct.BulkImport" weight="1000"/>
+ <edge id="ct.DeleteRange" weight="1000"/>
+ <edge id="ct.CreateUser" weight="1000"/>
+ <edge id="ct.DropUser" weight="1000"/>
+ <edge id="ct.ChangeAuthorizations" weight="1000"/>
+ <edge id="ct.ChangePermissions" weight="1000"/>
+ <edge id="ct.CheckPermission" weight="1000"/>
+ <edge id="ct.StopTabletServer" weight="100"/>
+ <edge id="ct.StartAll" weight="1000"/>
+ <edge id="ct.Shutdown" weight="10"/>
+ <edge id="ct.Config" weight="1000"/>
+ <edge id="ct.CreateNamespace" weight="1000"/>
+ <edge id="ct.DeleteNamespace" weight="100"/>
+ <edge id="ct.RenameNamespace" weight="100"/>
+ <edge id="ct.Apocalypse" weight="10"/>
+ <edge id="END" weight="1"/>
+</node>
+
+<node id="ct.Setup">
+ <property key="numTables" value="9"/>
+ <property key="numNamespaces" value="2"/>
+ <edge id="ct.CreateTable" weight="1"/>
+</node>
+
+<node id="ct.BatchScan">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.BatchWrite">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CloneTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CreateTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.DeleteTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.RenameTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ScanTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Replication">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.IsolatedScan">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.AddSplits">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ListSplits">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.OfflineTable">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Merge">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Compact">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.BulkImport">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.DeleteRange">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CreateUser">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.DropUser">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ChangeAuthorizations">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ChangePermissions">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CheckPermission">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.StopTabletServer">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.StartAll">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Config">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Shutdown">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.Apocalypse">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.CreateNamespace">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.DeleteNamespace">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.RenameNamespace">
+ <edge id="ct.StartAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Conditional.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Conditional.xml b/core/src/main/resources/randomwalk/modules/Conditional.xml
new file mode 100644
index 0000000..54ff7ab
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Conditional.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="ct" value="org.apache.accumulo.test.randomwalk.conditional"/>
+
+
+<init id="ct.Setup"/>
+
+<node id="dummy.ToAll">
+ <edge id="ct.Compact" weight="1"/>
+ <edge id="ct.Flush" weight="1"/>
+ <edge id="ct.Merge" weight="1"/>
+ <edge id="ct.Split" weight="1"/>
+ <edge id="ct.Transfer" weight="100000"/>
+ <edge id="ct.Verify" weight="2"/>
+</node>
+
+<node id="ct.Setup">
+ <property key="numAccts" value="10000"/>
+ <property key="numBanks" value="1000"/>
+ <edge id="ct.Init" weight="1"/>
+</node>
+
+<node id="ct.Init">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Compact">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Flush">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Merge">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Split">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Transfer">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Verify">
+ <edge id="dummy.ToAll" weight="1000"/>
+ <edge id="ct.TearDown" weight="1"/>
+</node>
+
+
+<node id="ct.TearDown">
+ <edge id="END" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Image.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Image.xml b/core/src/main/resources/randomwalk/modules/Image.xml
new file mode 100644
index 0000000..7561895
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Image.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<package prefix="image" value="org.apache.accumulo.test.randomwalk.image"/>
+
+<fixture id="image.ImageFixture"/>
+
+<init id="BigWrite"/>
+
+<node id="BigWrite" src="image.Write">
+ <property key="minSize" value="400000"/>
+ <property key="maxSize" value="1000000"/>
+ <edge id="BigWrite" weight="167"/>
+ <edge id="SmallWrite" weight="833"/>
+ <edge id="image.Commit" weight="1"/>
+</node>
+
+<node id="SmallWrite" src="image.Write">
+ <property key="minSize" value="4000"/>
+ <property key="maxSize" value="10000"/>
+ <edge id="BigWrite" weight="167"/>
+ <edge id="SmallWrite" weight="833"/>
+ <edge id="image.Commit" weight="1"/>
+</node>
+
+<node id="image.Commit">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="SmallWrite" weight="300"/>
+ <edge id="BigWrite" weight="300"/>
+ <edge id="image.Verify" weight="150"/>
+ <edge id="image.ScanMeta" weight="150"/>
+ <edge id="image.TableOp" weight="30"/>
+ <edge id="END" weight="1"/>
+</node>
+
+<node id="image.Verify">
+ <property key="maxVerify" value="500"/>
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="image.ScanMeta">
+ <property key="minScan" value="100"/>
+ <property key="maxScan" value="500"/>
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="image.TableOp">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/LongClean.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/LongClean.xml b/core/src/main/resources/randomwalk/modules/LongClean.xml
new file mode 100644
index 0000000..217afb0
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/LongClean.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<module>
+
+<init id="dummy.ToAll"/>
+
+<node id="Image.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+ <edge id="Image.xml" weight="1"/>
+ <edge id="Sequential.xml" weight="1"/>
+ <edge id="MultiTable.xml" weight="1"/>
+ <edge id="Shard.xml" weight="1"/>
+ <edge id="Concurrent.xml" weight="1"/>
+ <edge id="Security.xml" weight="1"/>
+ <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="MultiTable.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Shard.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Concurrent.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Security.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Bulk.xml" maxSec="3600" teardown="true">
+ <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>