You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2017/01/10 15:37:48 UTC
[2/9] accumulo git commit: ACCUMULO-4510 Removing random walk test
code
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteSomeDocs.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteSomeDocs.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteSomeDocs.java
deleted file mode 100644
index 08fcb4f..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteSomeDocs.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Properties;
-import java.util.Random;
-
-import org.apache.accumulo.core.client.BatchDeleter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.iterators.user.RegExFilter;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-//a test created to test the batch deleter
-public class DeleteSomeDocs extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- // delete documents that where the document id matches a given pattern from doc and index table
- // using the batch deleter
-
- Random rand = (Random) state.get("rand");
- String indexTableName = (String) state.get("indexTableName");
- String dataTableName = (String) state.get("docTableName");
-
- ArrayList<String> patterns = new ArrayList<>();
-
- for (Object key : props.keySet())
- if (key instanceof String && ((String) key).startsWith("pattern"))
- patterns.add(props.getProperty((String) key));
-
- String pattern = patterns.get(rand.nextInt(patterns.size()));
- BatchWriterConfig bwc = new BatchWriterConfig();
- BatchDeleter ibd = env.getConnector().createBatchDeleter(indexTableName, Authorizations.EMPTY, 8, bwc);
- ibd.setRanges(Collections.singletonList(new Range()));
-
- IteratorSetting iterSettings = new IteratorSetting(100, RegExFilter.class);
- RegExFilter.setRegexs(iterSettings, null, null, pattern, null, false);
-
- ibd.addScanIterator(iterSettings);
-
- ibd.delete();
-
- ibd.close();
-
- BatchDeleter dbd = env.getConnector().createBatchDeleter(dataTableName, Authorizations.EMPTY, 8, bwc);
- dbd.setRanges(Collections.singletonList(new Range()));
-
- iterSettings = new IteratorSetting(100, RegExFilter.class);
- RegExFilter.setRegexs(iterSettings, pattern, null, null, null, false);
-
- dbd.addScanIterator(iterSettings);
-
- dbd.delete();
-
- dbd.close();
-
- log.debug("Deleted documents w/ id matching '" + pattern + "'");
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteWord.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteWord.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteWord.java
deleted file mode 100644
index 9de0240..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/DeleteWord.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.ArrayList;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Random;
-
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-import org.apache.hadoop.io.Text;
-
-/**
- * Delete all documents containing a particular word.
- *
- */
-
-public class DeleteWord extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- String indexTableName = (String) state.get("indexTableName");
- String docTableName = (String) state.get("docTableName");
- int numPartitions = (Integer) state.get("numPartitions");
- Random rand = (Random) state.get("rand");
-
- String wordToDelete = Insert.generateRandomWord(rand);
-
- // use index to find all documents containing word
- Scanner scanner = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY);
- scanner.fetchColumnFamily(new Text(wordToDelete));
-
- ArrayList<Range> documentsToDelete = new ArrayList<>();
-
- for (Entry<Key,Value> entry : scanner)
- documentsToDelete.add(new Range(entry.getKey().getColumnQualifier()));
-
- if (documentsToDelete.size() > 0) {
- // use a batch scanner to fetch all documents
- BatchScanner bscanner = env.getConnector().createBatchScanner(docTableName, Authorizations.EMPTY, 8);
- bscanner.setRanges(documentsToDelete);
-
- BatchWriter ibw = env.getMultiTableBatchWriter().getBatchWriter(indexTableName);
- BatchWriter dbw = env.getMultiTableBatchWriter().getBatchWriter(docTableName);
-
- int count = 0;
-
- for (Entry<Key,Value> entry : bscanner) {
- String docID = entry.getKey().getRow().toString();
- String doc = entry.getValue().toString();
-
- Insert.unindexDocument(ibw, doc, docID, numPartitions);
-
- Mutation m = new Mutation(docID);
- m.putDelete("doc", "");
-
- dbw.addMutation(m);
- count++;
- }
-
- bscanner.close();
-
- env.getMultiTableBatchWriter().flush();
-
- if (count != documentsToDelete.size()) {
- throw new Exception("Batch scanner did not return expected number of docs " + count + " " + documentsToDelete.size());
- }
- }
-
- log.debug("Deleted " + documentsToDelete.size() + " documents containing " + wordToDelete);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
deleted file mode 100644
index 5b57ace..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ExportIndex.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map.Entry;
-import java.util.Properties;
-
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-
-/**
- *
- */
-public class ExportIndex extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
-
- String indexTableName = (String) state.get("indexTableName");
- String tmpIndexTableName = indexTableName + "_tmp";
-
- String exportDir = "/tmp/shard_export/" + indexTableName;
- String copyDir = "/tmp/shard_export/" + tmpIndexTableName;
-
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
-
- fs.delete(new Path("/tmp/shard_export/" + indexTableName), true);
- fs.delete(new Path("/tmp/shard_export/" + tmpIndexTableName), true);
-
- // disable spits, so that splits can be compared later w/o worrying one table splitting and the other not
- env.getConnector().tableOperations().setProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "20G");
-
- long t1 = System.currentTimeMillis();
-
- env.getConnector().tableOperations().flush(indexTableName, null, null, true);
- env.getConnector().tableOperations().offline(indexTableName);
-
- long t2 = System.currentTimeMillis();
-
- env.getConnector().tableOperations().exportTable(indexTableName, exportDir);
-
- long t3 = System.currentTimeMillis();
-
- // copy files
- BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(new Path(exportDir, "distcp.txt")), UTF_8));
- String file = null;
- while ((file = reader.readLine()) != null) {
- Path src = new Path(file);
- Path dest = new Path(new Path(copyDir), src.getName());
- FileUtil.copy(fs, src, fs, dest, false, true, CachedConfiguration.getInstance());
- }
-
- reader.close();
-
- long t4 = System.currentTimeMillis();
-
- env.getConnector().tableOperations().online(indexTableName);
- env.getConnector().tableOperations().importTable(tmpIndexTableName, copyDir);
-
- long t5 = System.currentTimeMillis();
-
- fs.delete(new Path(exportDir), true);
- fs.delete(new Path(copyDir), true);
-
- HashSet<Text> splits1 = new HashSet<>(env.getConnector().tableOperations().listSplits(indexTableName));
- HashSet<Text> splits2 = new HashSet<>(env.getConnector().tableOperations().listSplits(tmpIndexTableName));
-
- if (!splits1.equals(splits2))
- throw new Exception("Splits not equals " + indexTableName + " " + tmpIndexTableName);
-
- HashMap<String,String> props1 = new HashMap<>();
- for (Entry<String,String> entry : env.getConnector().tableOperations().getProperties(indexTableName))
- props1.put(entry.getKey(), entry.getValue());
-
- HashMap<String,String> props2 = new HashMap<>();
- for (Entry<String,String> entry : env.getConnector().tableOperations().getProperties(tmpIndexTableName))
- props2.put(entry.getKey(), entry.getValue());
-
- if (!props1.equals(props2))
- throw new Exception("Props not equals " + indexTableName + " " + tmpIndexTableName);
-
- // unset the split threshold
- env.getConnector().tableOperations().removeProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey());
- env.getConnector().tableOperations().removeProperty(tmpIndexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey());
-
- log.debug("Imported " + tmpIndexTableName + " from " + indexTableName + " flush: " + (t2 - t1) + "ms export: " + (t3 - t2) + "ms copy:" + (t4 - t3)
- + "ms import:" + (t5 - t4) + "ms");
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Flush.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Flush.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Flush.java
deleted file mode 100644
index 39d32c4..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Flush.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.Properties;
-import java.util.Random;
-
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-public class Flush extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- String indexTableName = (String) state.get("indexTableName");
- String dataTableName = (String) state.get("docTableName");
- Random rand = (Random) state.get("rand");
-
- String table;
-
- if (rand.nextDouble() < .5)
- table = indexTableName;
- else
- table = dataTableName;
-
- env.getConnector().tableOperations().flush(table, null, null, true);
- log.debug("Flushed " + table);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Grep.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Grep.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Grep.java
deleted file mode 100644
index 7409ea7..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Grep.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Random;
-
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.user.IntersectingIterator;
-import org.apache.accumulo.core.iterators.user.RegExFilter;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-import org.apache.hadoop.io.Text;
-
-public class Grep extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- // pick a few randoms words... grep for those words and search the index
- // ensure both return the same set of documents
-
- String indexTableName = (String) state.get("indexTableName");
- String dataTableName = (String) state.get("docTableName");
- Random rand = (Random) state.get("rand");
-
- Text words[] = new Text[rand.nextInt(4) + 2];
-
- for (int i = 0; i < words.length; i++) {
- words[i] = new Text(Insert.generateRandomWord(rand));
- }
-
- BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 16);
- IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class.getName());
- IntersectingIterator.setColumnFamilies(ii, words);
- bs.addScanIterator(ii);
- bs.setRanges(Collections.singleton(new Range()));
-
- HashSet<Text> documentsFoundInIndex = new HashSet<>();
-
- for (Entry<Key,Value> entry2 : bs) {
- documentsFoundInIndex.add(entry2.getKey().getColumnQualifier());
- }
-
- bs.close();
-
- bs = env.getConnector().createBatchScanner(dataTableName, Authorizations.EMPTY, 16);
-
- for (int i = 0; i < words.length; i++) {
- IteratorSetting more = new IteratorSetting(20 + i, "ii" + i, RegExFilter.class);
- RegExFilter.setRegexs(more, null, null, null, "(^|(.*\\s))" + words[i] + "($|(\\s.*))", false);
- bs.addScanIterator(more);
- }
-
- bs.setRanges(Collections.singleton(new Range()));
-
- HashSet<Text> documentsFoundByGrep = new HashSet<>();
-
- for (Entry<Key,Value> entry2 : bs) {
- documentsFoundByGrep.add(entry2.getKey().getRow());
- }
-
- bs.close();
-
- if (!documentsFoundInIndex.equals(documentsFoundByGrep)) {
- throw new Exception("Set of documents found not equal for words " + Arrays.asList(words).toString() + " " + documentsFoundInIndex + " "
- + documentsFoundByGrep);
- }
-
- log.debug("Grep and index agree " + Arrays.asList(words).toString() + " " + documentsFoundInIndex.size());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Insert.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Insert.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Insert.java
deleted file mode 100644
index 8482a59..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Insert.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.HashSet;
-import java.util.Properties;
-import java.util.Random;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-public class Insert extends Test {
-
- static final int NUM_WORDS = 100000;
- static final int MIN_WORDS_PER_DOC = 10;
- static final int MAX_WORDS_PER_DOC = 3000;
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- String indexTableName = (String) state.get("indexTableName");
- String dataTableName = (String) state.get("docTableName");
- int numPartitions = (Integer) state.get("numPartitions");
- Random rand = (Random) state.get("rand");
- long nextDocID = (Long) state.get("nextDocID");
-
- BatchWriter dataWriter = env.getMultiTableBatchWriter().getBatchWriter(dataTableName);
- BatchWriter indexWriter = env.getMultiTableBatchWriter().getBatchWriter(indexTableName);
-
- String docID = insertRandomDocument(nextDocID++, dataWriter, indexWriter, indexTableName, dataTableName, numPartitions, rand);
-
- log.debug("Inserted document " + docID);
-
- state.set("nextDocID", Long.valueOf(nextDocID));
- }
-
- static String insertRandomDocument(long did, BatchWriter dataWriter, BatchWriter indexWriter, String indexTableName, String dataTableName, int numPartitions,
- Random rand) throws TableNotFoundException, Exception, AccumuloException, AccumuloSecurityException {
- String doc = createDocument(rand);
-
- String docID = new StringBuilder(String.format("%016x", did)).reverse().toString();
-
- saveDocument(dataWriter, docID, doc);
- indexDocument(indexWriter, doc, docID, numPartitions);
-
- return docID;
- }
-
- static void saveDocument(BatchWriter bw, String docID, String doc) throws Exception {
-
- Mutation m = new Mutation(docID);
- m.put("doc", "", doc);
-
- bw.addMutation(m);
- }
-
- static String createDocument(Random rand) {
- StringBuilder sb = new StringBuilder();
-
- int numWords = rand.nextInt(MAX_WORDS_PER_DOC - MIN_WORDS_PER_DOC) + MIN_WORDS_PER_DOC;
-
- for (int i = 0; i < numWords; i++) {
- String word = generateRandomWord(rand);
-
- if (i > 0)
- sb.append(" ");
-
- sb.append(word);
- }
-
- return sb.toString();
- }
-
- static String generateRandomWord(Random rand) {
- return Integer.toString(rand.nextInt(NUM_WORDS), Character.MAX_RADIX);
- }
-
- static String genPartition(int partition) {
- return String.format("%06x", Math.abs(partition));
- }
-
- static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception {
- indexDocument(bw, doc, docId, numPartitions, false);
- }
-
- static void unindexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception {
- indexDocument(bw, doc, docId, numPartitions, true);
- }
-
- static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions, boolean delete) throws Exception {
-
- String[] tokens = doc.split("\\W+");
-
- String partition = genPartition(doc.hashCode() % numPartitions);
-
- Mutation m = new Mutation(partition);
-
- HashSet<String> tokensSeen = new HashSet<>();
-
- for (String token : tokens) {
- token = token.toLowerCase();
-
- if (!tokensSeen.contains(token)) {
- tokensSeen.add(token);
- if (delete)
- m.putDelete(token, docId);
- else
- m.put(token, docId, new Value(new byte[0]));
- }
- }
-
- if (m.size() > 0)
- bw.addMutation(m);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Merge.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Merge.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Merge.java
deleted file mode 100644
index 910e64c..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Merge.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.Collection;
-import java.util.Properties;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-import org.apache.hadoop.io.Text;
-
-public class Merge extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- String indexTableName = (String) state.get("indexTableName");
-
- Collection<Text> splits = env.getConnector().tableOperations().listSplits(indexTableName);
- SortedSet<Text> splitSet = new TreeSet<>(splits);
- log.debug("merging " + indexTableName);
- env.getConnector().tableOperations().merge(indexTableName, null, null);
- org.apache.accumulo.core.util.Merge merge = new org.apache.accumulo.core.util.Merge();
- merge.mergomatic(env.getConnector(), indexTableName, null, null, 256 * 1024 * 1024, true);
- splits = env.getConnector().tableOperations().listSplits(indexTableName);
- if (splits.size() > splitSet.size()) {
- // throw an excpetion so that test will die an no further changes to table will occur...
- // this way table is left as is for debugging.
- throw new Exception("There are more tablets after a merge: " + splits.size() + " was " + splitSet.size());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Reindex.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Reindex.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Reindex.java
deleted file mode 100644
index 5aed6e3..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Reindex.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Random;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-public class Reindex extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- String indexTableName = (String) state.get("indexTableName");
- String tmpIndexTableName = indexTableName + "_tmp";
- String docTableName = (String) state.get("docTableName");
- int numPartitions = (Integer) state.get("numPartitions");
-
- Random rand = (Random) state.get("rand");
-
- ShardFixture.createIndexTable(this.log, state, env, "_tmp", rand);
-
- Scanner scanner = env.getConnector().createScanner(docTableName, Authorizations.EMPTY);
- BatchWriter tbw = env.getConnector().createBatchWriter(tmpIndexTableName, new BatchWriterConfig());
-
- int count = 0;
-
- for (Entry<Key,Value> entry : scanner) {
- String docID = entry.getKey().getRow().toString();
- String doc = entry.getValue().toString();
-
- Insert.indexDocument(tbw, doc, docID, numPartitions);
-
- count++;
- }
-
- tbw.close();
-
- log.debug("Reindexed " + count + " documents into " + tmpIndexTableName);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Search.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Search.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Search.java
deleted file mode 100644
index bccb280..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Search.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Random;
-
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.user.IntersectingIterator;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-import org.apache.hadoop.io.Text;
-
-public class Search extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- String indexTableName = (String) state.get("indexTableName");
- String dataTableName = (String) state.get("docTableName");
-
- Random rand = (Random) state.get("rand");
-
- Entry<Key,Value> entry = findRandomDocument(state, env, dataTableName, rand);
- if (entry == null)
- return;
-
- Text docID = entry.getKey().getRow();
- String doc = entry.getValue().toString();
-
- String[] tokens = doc.split("\\W+");
- int numSearchTerms = rand.nextInt(6);
- if (numSearchTerms < 2)
- numSearchTerms = 2;
-
- HashSet<String> searchTerms = new HashSet<>();
- while (searchTerms.size() < numSearchTerms)
- searchTerms.add(tokens[rand.nextInt(tokens.length)]);
-
- Text columns[] = new Text[searchTerms.size()];
- int index = 0;
- for (String term : searchTerms) {
- columns[index++] = new Text(term);
- }
-
- log.debug("Looking up terms " + searchTerms + " expect to find " + docID);
-
- BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 10);
- IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class);
- IntersectingIterator.setColumnFamilies(ii, columns);
- bs.addScanIterator(ii);
- bs.setRanges(Collections.singleton(new Range()));
-
- boolean sawDocID = false;
-
- for (Entry<Key,Value> entry2 : bs) {
- if (entry2.getKey().getColumnQualifier().equals(docID)) {
- sawDocID = true;
- break;
- }
- }
-
- bs.close();
-
- if (!sawDocID)
- throw new Exception("Did not see doc " + docID + " in index. terms:" + searchTerms + " " + indexTableName + " " + dataTableName);
- }
-
- static Entry<Key,Value> findRandomDocument(State state, Environment env, String dataTableName, Random rand) throws Exception {
- Scanner scanner = env.getConnector().createScanner(dataTableName, Authorizations.EMPTY);
- scanner.setBatchSize(1);
- scanner.setRange(new Range(Integer.toString(rand.nextInt(0xfffffff), 16), null));
-
- Iterator<Entry<Key,Value>> iter = scanner.iterator();
- if (!iter.hasNext())
- return null;
-
- return iter.next();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ShardFixture.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ShardFixture.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ShardFixture.java
deleted file mode 100644
index 99e3a61..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/ShardFixture.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.net.InetAddress;
-import java.util.Random;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MultiTableBatchWriter;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.Fixture;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
-public class ShardFixture extends Fixture {
-
- static SortedSet<Text> genSplits(long max, int numTablets, String format) {
-
- int numSplits = numTablets - 1;
- long distance = max / numTablets;
- long split = distance;
-
- TreeSet<Text> splits = new TreeSet<>();
-
- for (int i = 0; i < numSplits; i++) {
- splits.add(new Text(String.format(format, split)));
- split += distance;
- }
-
- return splits;
- }
-
- static void createIndexTable(Logger log, State state, Environment env, String suffix, Random rand) throws Exception {
- Connector conn = env.getConnector();
- String name = (String) state.get("indexTableName") + suffix;
- int numPartitions = (Integer) state.get("numPartitions");
- boolean enableCache = (Boolean) state.get("cacheIndex");
- conn.tableOperations().create(name);
-
- String tableId = conn.tableOperations().tableIdMap().get(name);
- log.info("Created index table " + name + "(id:" + tableId + ")");
-
- SortedSet<Text> splits = genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x");
- conn.tableOperations().addSplits(name, splits);
-
- log.info("Added " + splits.size() + " splits to " + name);
-
- if (enableCache) {
- conn.tableOperations().setProperty(name, Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
- conn.tableOperations().setProperty(name, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
-
- log.info("Enabled caching for table " + name);
- }
- }
-
- @Override
- public void setUp(State state, Environment env) throws Exception {
- String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
- String pid = env.getPid();
-
- Random rand = new Random();
-
- int numPartitions = rand.nextInt(90) + 10;
-
- state.set("indexTableName", String.format("ST_index_%s_%s_%d", hostname, pid, System.currentTimeMillis()));
- state.set("docTableName", String.format("ST_docs_%s_%s_%d", hostname, pid, System.currentTimeMillis()));
- state.set("numPartitions", Integer.valueOf(numPartitions));
- state.set("cacheIndex", rand.nextDouble() < .5);
- state.set("rand", rand);
- state.set("nextDocID", Long.valueOf(0));
-
- Connector conn = env.getConnector();
-
- createIndexTable(this.log, state, env, "", rand);
-
- String docTableName = (String) state.get("docTableName");
- conn.tableOperations().create(docTableName);
-
- String tableId = conn.tableOperations().tableIdMap().get(docTableName);
- log.info("Created doc table " + docTableName + " (id:" + tableId + ")");
-
- SortedSet<Text> splits = genSplits(0xff, rand.nextInt(32) + 1, "%02x");
- conn.tableOperations().addSplits(docTableName, splits);
-
- log.info("Added " + splits.size() + " splits to " + docTableName);
-
- if (rand.nextDouble() < .5) {
- conn.tableOperations().setProperty((String) state.get("docTableName"), Property.TABLE_BLOOM_ENABLED.getKey(), "true");
- log.info("Enabled bloom filters for table " + (String) state.get("docTableName"));
- }
- }
-
- @Override
- public void tearDown(State state, Environment env) throws Exception {
- // We have resources we need to clean up
- if (env.isMultiTableBatchWriterInitialized()) {
- MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
- try {
- mtbw.close();
- } catch (MutationsRejectedException e) {
- log.error("Ignoring mutations that weren't flushed", e);
- }
-
- // Reset the MTBW on the state to null
- env.resetMultiTableBatchWriter();
- }
-
- Connector conn = env.getConnector();
-
- log.info("Deleting index and doc tables");
-
- conn.tableOperations().delete((String) state.get("indexTableName"));
- conn.tableOperations().delete((String) state.get("docTableName"));
-
- log.debug("Exiting shard test");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/SortTool.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/SortTool.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/SortTool.java
deleted file mode 100644
index cdeccb3..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/SortTool.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.Collection;
-
-import org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat;
-import org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.util.Tool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SortTool extends Configured implements Tool {
- protected final Logger log = LoggerFactory.getLogger(this.getClass());
- private String outputDir;
- private String seqFile;
- private String splitFile;
- private Collection<Text> splits;
-
- public SortTool(String seqFile, String outputDir, String splitFile, Collection<Text> splits) {
- this.outputDir = outputDir;
- this.seqFile = seqFile;
- this.splitFile = splitFile;
- this.splits = splits;
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Job job = Job.getInstance(getConf(), this.getClass().getSimpleName());
- job.setJarByClass(this.getClass());
-
- if (job.getJar() == null) {
- log.error("M/R requires a jar file! Run mvn package.");
- return 1;
- }
-
- job.setInputFormatClass(SequenceFileInputFormat.class);
- SequenceFileInputFormat.setInputPaths(job, seqFile);
-
- job.setPartitionerClass(KeyRangePartitioner.class);
- KeyRangePartitioner.setSplitFile(job, splitFile);
-
- job.setMapOutputKeyClass(Key.class);
- job.setMapOutputValueClass(Value.class);
-
- job.setNumReduceTasks(splits.size() + 1);
-
- job.setOutputFormatClass(AccumuloFileOutputFormat.class);
- AccumuloFileOutputFormat.setOutputPath(job, new Path(outputDir));
-
- job.waitForCompletion(true);
- return job.isSuccessful() ? 0 : 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Split.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Split.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Split.java
deleted file mode 100644
index 0420fb8..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/Split.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.Properties;
-import java.util.Random;
-import java.util.SortedSet;
-
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-import org.apache.hadoop.io.Text;
-
-public class Split extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
- String indexTableName = (String) state.get("indexTableName");
- int numPartitions = (Integer) state.get("numPartitions");
- Random rand = (Random) state.get("rand");
-
- SortedSet<Text> splitSet = ShardFixture.genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x");
- log.debug("adding splits " + indexTableName);
- env.getConnector().tableOperations().addSplits(indexTableName, splitSet);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/VerifyIndex.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/VerifyIndex.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/VerifyIndex.java
deleted file mode 100644
index 70440d8..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/shard/VerifyIndex.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.shard;
-
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.Properties;
-
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-public class VerifyIndex extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {
-
- String indexTableName = (String) state.get("indexTableName");
- String tmpIndexTableName = indexTableName + "_tmp";
-
- // scan new and old index and verify identical
- Scanner indexScanner1 = env.getConnector().createScanner(tmpIndexTableName, Authorizations.EMPTY);
- Scanner indexScanner2 = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY);
-
- Iterator<Entry<Key,Value>> iter = indexScanner2.iterator();
-
- int count = 0;
-
- for (Entry<Key,Value> entry : indexScanner1) {
- if (!iter.hasNext())
- throw new Exception("index rebuild mismatch " + entry.getKey() + " " + indexTableName);
-
- Key key1 = entry.getKey();
- Key key2 = iter.next().getKey();
-
- if (!key1.equals(key2, PartialKey.ROW_COLFAM_COLQUAL))
- throw new Exception("index rebuild mismatch " + key1 + " " + key2 + " " + indexTableName + " " + tmpIndexTableName);
- count++;
- if (count % 1000 == 0)
- makingProgress();
- }
-
- if (iter.hasNext())
- throw new Exception("index rebuild mismatch " + iter.next().getKey() + " " + tmpIndexTableName);
-
- log.debug("Verified " + count + " index entries ");
-
- env.getConnector().tableOperations().delete(indexTableName);
- env.getConnector().tableOperations().rename(tmpIndexTableName, indexTableName);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/CreateTable.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/CreateTable.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/CreateTable.java
deleted file mode 100644
index 2e14de7..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/CreateTable.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.accumulo.test.randomwalk.unit;
-
-import java.util.Properties;
-
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-public class CreateTable extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {}
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/DeleteTable.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/DeleteTable.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/DeleteTable.java
deleted file mode 100644
index c0b7e65..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/DeleteTable.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.unit;
-
-import java.util.Properties;
-
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-public class DeleteTable extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {}
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Ingest.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Ingest.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Ingest.java
deleted file mode 100644
index 40f3555..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Ingest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.unit;
-
-import java.util.Properties;
-
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-public class Ingest extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {}
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Scan.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Scan.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Scan.java
deleted file mode 100644
index 5273ea8..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Scan.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.unit;
-
-import java.util.Properties;
-
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-public class Scan extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {}
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Verify.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Verify.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Verify.java
deleted file mode 100644
index c41d633..0000000
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/unit/Verify.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk.unit;
-
-import java.util.Properties;
-
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.State;
-import org.apache.accumulo.test.randomwalk.Test;
-
-public class Verify extends Test {
-
- @Override
- public void visit(State state, Environment env, Properties props) throws Exception {}
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationRandomWalkIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationRandomWalkIT.java b/test/src/main/java/org/apache/accumulo/test/replication/ReplicationRandomWalkIT.java
deleted file mode 100644
index 80bc69d..0000000
--- a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationRandomWalkIT.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.replication;
-
-import static org.apache.accumulo.core.conf.Property.TSERV_ARCHIVE_WALOGS;
-import static org.apache.accumulo.core.conf.Property.TSERV_WALOG_MAX_SIZE;
-
-import java.util.Properties;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.accumulo.test.randomwalk.Environment;
-import org.apache.accumulo.test.randomwalk.concurrent.Replication;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-public class ReplicationRandomWalkIT extends ConfigurableMacBase {
-
- @Override
- protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(TSERV_ARCHIVE_WALOGS, "false");
- cfg.setProperty(TSERV_WALOG_MAX_SIZE, "1M");
- cfg.setNumTservers(1);
- }
-
- @Test(timeout = 5 * 60 * 1000)
- public void runReplicationRandomWalkStep() throws Exception {
- Replication r = new Replication();
-
- Environment env = new Environment(new Properties()) {
- @Override
- public String getUserName() {
- return "root";
- }
-
- @Override
- public String getPassword() {
- return ROOT_PASSWORD;
- }
-
- @Override
- public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
- return ReplicationRandomWalkIT.this.getConnector();
- }
-
- };
- r.visit(null, env, null);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/resources/randomwalk/module.xsd
----------------------------------------------------------------------
diff --git a/test/src/main/resources/randomwalk/module.xsd b/test/src/main/resources/randomwalk/module.xsd
deleted file mode 100644
index bcdaaae0..0000000
--- a/test/src/main/resources/randomwalk/module.xsd
+++ /dev/null
@@ -1,69 +0,0 @@
-<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
- <xsd:element name="module" type="ModuleType"/>
-
- <xsd:complexType name="ModuleType">
- <xsd:sequence>
- <xsd:element name="package" type="PrefixType" minOccurs="0" maxOccurs="unbounded"/>
- <xsd:element name="fixture" type="InitType" minOccurs="0" maxOccurs="1"/>
- <xsd:element name="init" type="InitType"/>
- <xsd:element name="node" type="NodeType" minOccurs="1" maxOccurs="unbounded"/>
- </xsd:sequence>
- </xsd:complexType>
-
- <xsd:complexType name="PrefixType">
- <xsd:attribute name="prefix" type="xsd:string"/>
- <xsd:attribute name="value" type="xsd:string"/>
- </xsd:complexType>
-
- <xsd:complexType name="InitType">
- <xsd:attribute name="id" type="xsd:string"/>
- <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
- <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
- <xsd:attribute name="teardown" type="xsd:boolean"/>
- </xsd:complexType>
-
- <xsd:complexType name="NodeType">
- <xsd:sequence>
- <xsd:element name="alias" type="AliasType" minOccurs="0" maxOccurs="unbounded"/>
- <xsd:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
- <xsd:element name="edge" type="EdgeType" minOccurs="1" maxOccurs="unbounded"/>
- </xsd:sequence>
- <xsd:attribute name="id" type="xsd:string"/>
- <xsd:attribute name="src" type="xsd:string"/>
- <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
- <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
- <xsd:attribute name="teardown" type="xsd:boolean"/>
- </xsd:complexType>
-
- <xsd:complexType name="EdgeType">
- <xsd:attribute name="id" type="xsd:string"/>
- <xsd:attribute name="weight" type="xsd:positiveInteger"/>
- </xsd:complexType>
-
- <xsd:complexType name="AliasType">
- <xsd:attribute name="name" type="xsd:string"/>
- </xsd:complexType>
-
- <xsd:complexType name="PropertyType">
- <xsd:attribute name="key" type="xsd:string"/>
- <xsd:attribute name="value" type="xsd:string"/>
- </xsd:complexType>
-
-</xsd:schema>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/resources/unit/Basic.xml
----------------------------------------------------------------------
diff --git a/test/src/main/resources/unit/Basic.xml b/test/src/main/resources/unit/Basic.xml
deleted file mode 100644
index 2dead02..0000000
--- a/test/src/main/resources/unit/Basic.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<module>
-
-<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
-
-<init id="test.CreateTable"/>
-
-<node id="test.CreateTable">
- <edge id="unit/Simple.xml" weight="1"/>
-</node>
-
-<node id="unit/Simple.xml">
- <edge id="unit/Simple.xml" weight="3"/>
- <edge id="test.DeleteTable" weight="1"/>
-</node>
-
-<node id="test.DeleteTable">
- <edge id="END" weight="1"/>
-</node>
-
-</module>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/main/resources/unit/Simple.xml
----------------------------------------------------------------------
diff --git a/test/src/main/resources/unit/Simple.xml b/test/src/main/resources/unit/Simple.xml
deleted file mode 100644
index cad940e..0000000
--- a/test/src/main/resources/unit/Simple.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<module>
-
-<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
-
-<init id="dummy.all"/>
-
-<node id="dummy.all">
- <edge id="test.Ingest" weight="1"/>
- <edge id="test.Verify" weight="1"/>
- <edge id="test.Scan" weight="1"/>
- <edge id="END" weight="1"/>
-</node>
-
-<node id="test.Ingest">
- <edge id="dummy.all" weight="1"/>
-</node>
-
-<node id="test.Verify">
- <edge id="dummy.all" weight="1"/>
-</node>
-
-<node id="test.Scan">
- <edge id="dummy.all" weight="1"/>
-</node>
-
-</module>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java b/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java
deleted file mode 100644
index ca003ab..0000000
--- a/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.randomwalk;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-import javax.xml.XMLConstants;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.validation.Schema;
-import javax.xml.validation.SchemaFactory;
-
-import org.apache.accumulo.test.randomwalk.unit.CreateTable;
-import org.w3c.dom.Document;
-import org.xml.sax.SAXException;
-
-public class FrameworkTest {
-
- // Need to use fully qualified name here because of conflict with org.apache.accumulo.test.randomwalk.Test
- @org.junit.Test
- public void testXML() throws SAXException, URISyntaxException, ParserConfigurationException, IOException {
- SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
- Schema moduleSchema = sf.newSchema(getFile("/randomwalk/module.xsd"));
-
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- dbf.setSchema(moduleSchema);
-
- DocumentBuilder docbuilder = dbf.newDocumentBuilder();
- Document document = docbuilder.parse(getFile("/randomwalk/Basic.xml"));
-
- assertNotEquals("Parsing randomwalk xml should result in nodes.", 0, document.getChildNodes().getLength());
- }
-
- private File getFile(String resource) throws URISyntaxException {
- return new File(this.getClass().getResource(resource).toURI());
- }
-
- @org.junit.Test
- public void testRWTest() {
- Test t1 = new CreateTable();
- assertEquals("org.apache.accumulo.test.randomwalk.unit.CreateTable", t1.toString());
-
- Test t2 = new CreateTable();
- assertEquals("CreateTable test nodes were not equal.", t1, t2);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/test/resources/randomwalk/Basic.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/randomwalk/Basic.xml b/test/src/test/resources/randomwalk/Basic.xml
deleted file mode 100644
index 2dead02..0000000
--- a/test/src/test/resources/randomwalk/Basic.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<module>
-
-<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
-
-<init id="test.CreateTable"/>
-
-<node id="test.CreateTable">
- <edge id="unit/Simple.xml" weight="1"/>
-</node>
-
-<node id="unit/Simple.xml">
- <edge id="unit/Simple.xml" weight="3"/>
- <edge id="test.DeleteTable" weight="1"/>
-</node>
-
-<node id="test.DeleteTable">
- <edge id="END" weight="1"/>
-</node>
-
-</module>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/src/test/resources/randomwalk/Simple.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/randomwalk/Simple.xml b/test/src/test/resources/randomwalk/Simple.xml
deleted file mode 100644
index cad940e..0000000
--- a/test/src/test/resources/randomwalk/Simple.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<module>
-
-<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
-
-<init id="dummy.all"/>
-
-<node id="dummy.all">
- <edge id="test.Ingest" weight="1"/>
- <edge id="test.Verify" weight="1"/>
- <edge id="test.Scan" weight="1"/>
- <edge id="END" weight="1"/>
-</node>
-
-<node id="test.Ingest">
- <edge id="dummy.all" weight="1"/>
-</node>
-
-<node id="test.Verify">
- <edge id="dummy.all" weight="1"/>
-</node>
-
-<node id="test.Scan">
- <edge id="dummy.all" weight="1"/>
-</node>
-
-</module>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/README.md
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/README.md b/test/system/randomwalk/README.md
deleted file mode 100644
index 786a160..0000000
--- a/test/system/randomwalk/README.md
+++ /dev/null
@@ -1,99 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-Apache Accumulo Random Walk Tests
-=================================
-
-The randomwalk framework needs to be configured for your Accumulo instance by
-doing the following steps:
-
-1. Make sure you have both `ACCUMULO_HOME` and `HADOOP_HOME` set in your
- `$ACCUMULO_CONF_DIR/accumulo-env.sh`.
-
-2. Create 'randomwalk.conf' file in the `conf` directory containing settings
- needed by walkers to connect to Accumulo.
-
-3. Create a 'walkers' file in the `conf` directory containing the hostnames of
- the machines where you want random walkers to run.
-
-3. Create a 'logger.xml' file in the `conf` directory from `logger.xml.example`.
-
-The command below starts random walkers on all machines listed in 'walkers'.
-The argument `Image.xml` indicates the module to use (which is located at
-`conf/modules/Image.xml`):
-
-> `$ ./bin/start-all.sh Image.xml`
-
-All modules must be in `conf/modules` and can be referenced without this prefix.
-For example, a module located at `conf/modules/foo/bar.xml` is started as
-the following:
-
-> `$ ./bin/start-all.sh foo/bar.xml`
-
-This command will load all configuration in the `conf` directory to HDFS and
-start identical random walkers on each node. These random walkers will
-download the current configuration from HDFS and place them in the `tmp/`
-directory.
-
-Random walkers will drop their logs in the `logs/` directory. If you are running
-multiple walkers and want ERROR/WARNs dropped to an NFS-hosted log, please set
-`NFS_LOGPATH` to a NFS-mounted directory and uncomment the NFS appender in `logger.xml`.
-
-You can kill all walkers on the machines listed in the 'walkers' file using
-the following command:
-
-> `$ ./bin/kill-all.sh`
-
-Module-Specific Configuration
------------------------------
-
-The user accounts for walkers that run the Concurrent.xml module must have
-password-less SSH access to the entire Accumulo cluster, so that they may run
-`$ACCUMULO_HOME/bin/start-all.sh`. Note that this is not the same script as the
-one that starts random walkers; it is the script to start up an Accumulo
-cluster. You can test that access is in place by running
-`$ACCUMULO_HOME/bin/start-all.sh` from the command line of each walker account.
-
-The above access is also needed for any modules that include Concurrent.xml,
-e.g., ShortClean.xml, LongClean.xml.
-
-Other Useful Commands
----------------------
-
-Copies configuration in `conf/` to HDFS:
-
-> `$ copy-config.sh`
-
-Copies configuration from HDFS into `tmp/` and starts only one local random walker.
-
-> `$ start-local.sh All.xml`
-
-Stops all local random walkers:
-
-> `$ pkill -f randomwalk.Framework`
-
-Known Issues
-------------
-
-If you are running randomwalk tests while exercising Hadoop's high availability
-(HA) failover capabilities, you should use Hadoop version 2.1.0 or later.
-Failover scenarios are more likely to cause randomwalk test failures under
-earlier Hadoop versions. See the following issue reports for more details.
-
-* [HDFS-4404](https://issues.apache.org/jira/browse/HDFS-4404)
-* [HADOOP-9792](https://issues.apache.org/jira/browse/HADOOP-9792)
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/bin/apocalypse.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/apocalypse.sh b/test/system/randomwalk/bin/apocalypse.sh
deleted file mode 100755
index e62b008..0000000
--- a/test/system/randomwalk/bin/apocalypse.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#copied below from hadoop-config.sh
-this="$0"
-while [[ -h "$this" ]]; do
- ls=$(ls -ld "$this")
- link=$(expr "$ls" : '.*-> \(.*\)$')
- if expr "$link" : '.*/.*' > /dev/null; then
- this="$link"
- else
- this=$(dirname "$this")/"$link"
- fi
-done
-bin=$(dirname "$this")
-script=$(basename "$this")
-bin=$(cd "$bin"; pwd)
-this="$bin/$script"
-
-ACCUMULO_HOME=$(dirname "$this")/../../../..
-export ACCUMULO_HOME=$(cd "$ACCUMULO_HOME"; pwd)
-
-if [[ -f $ACCUMULO_HOME/conf/accumulo-env.sh ]] ; then
- . "$ACCUMULO_HOME/conf/accumulo-env.sh"
-fi
-
-if [[ -z $HADOOP_PREFIX ]] ; then
- echo "HADOOP_PREFIX is not set. Please make sure it's set globally."
- exit 1
-fi
-
-echo 'killing all accumulo processes'
-pssh -h "$ACCUMULO_HOME/conf/tservers" "pkill -9 -f app=[tmg].*org.apache.accumulo.start " < /dev/null
-pssh -h "$ACCUMULO_HOME/conf/masters" "pkill -9 -f app=[tmg].*org.apache.accumulo.start " < /dev/null
-exit 0
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/bin/copy-config.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/copy-config.sh b/test/system/randomwalk/bin/copy-config.sh
deleted file mode 100755
index 48afc19..0000000
--- a/test/system/randomwalk/bin/copy-config.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-if [[ -z $HADOOP_PREFIX ]] ; then
- echo "HADOOP_PREFIX is not set. Please make sure it's set globally."
- exit 1
-fi
-
-if [[ -z $ACCUMULO_HOME ]] ; then
- echo "ACCUMULO_HOME is not set. Please make sure it's set globally."
- exit 1
-fi
-
-RW_HOME=$ACCUMULO_HOME/test/system/randomwalk
-
-cd "$RW_HOME"
-
-tar czf config.tgz conf
-"$HADOOP_PREFIX/bin/hadoop" fs -rmr /randomwalk 2>/dev/null
-"$HADOOP_PREFIX/bin/hadoop" fs -mkdir /randomwalk
-"$HADOOP_PREFIX/bin/hadoop" fs -put config.tgz /randomwalk
-"$HADOOP_PREFIX/bin/hadoop" fs -setrep 3 /randomwalk/config.tgz
-rm config.tgz
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/bin/kill-all.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/kill-all.sh b/test/system/randomwalk/bin/kill-all.sh
deleted file mode 100755
index e4bc0be..0000000
--- a/test/system/randomwalk/bin/kill-all.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#copied below from hadoop-config.sh
-this="$0"
-while [[ -h "$this" ]]; do
- ls=$(ls -ld "$this")
- link=$(expr "$ls" : '.*-> \(.*\)$')
- if expr "$link" : '.*/.*' > /dev/null; then
- this="$link"
- else
- this=$(dirname "$this")/"$link"
- fi
-done
-bin=$(dirname "$this")
-script=$(basename "$this")
-bin=$(cd "$bin"; pwd)
-this="$bin/$script"
-
-ACCUMULO_HOME=$(dirname "$this")/../../../..
-export ACCUMULO_HOME=$(cd "$ACCUMULO_HOME"; pwd)
-
-RW_HOME=$ACCUMULO_HOME/test/system/randomwalk
-
-echo 'killing random walkers'
-pssh -h "$RW_HOME/conf/walkers" "pkill -f [r]andomwalk.Framework" < /dev/null
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b24c338a/test/system/randomwalk/bin/kill-local.sh
----------------------------------------------------------------------
diff --git a/test/system/randomwalk/bin/kill-local.sh b/test/system/randomwalk/bin/kill-local.sh
deleted file mode 100755
index 2b35437..0000000
--- a/test/system/randomwalk/bin/kill-local.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-pkill -f randomwalk.Framework