You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/11/25 21:39:01 UTC
[1/3] git commit: ACCUMULO-1878 handle error conditions in example
tests.
Updated Branches:
refs/heads/1.6.0-SNAPSHOT 919691010 -> fd652ca0d
ACCUMULO-1878 handle error conditions in example tests.
* ensure simple examples used in integration tests properly set non-zero exit on errors
* check return code of executed commands.
* fix typo in bloom filter speed comparison
* fix error in arg order on helloworld examples
* Makes sure paired examples for RandomBatch use the same seed.
Signed-off-by: Keith Turner <kt...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1fe5f7f5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1fe5f7f5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1fe5f7f5
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 1fe5f7f528d39746c0e5cffb8710b4db21c3a5b2
Parents: d0ceebb
Author: Sean Busbey <bu...@clouderagovt.com>
Authored: Thu Nov 7 09:07:22 2013 -0600
Committer: Keith Turner <kt...@apache.org>
Committed: Mon Nov 25 09:36:39 2013 -0500
----------------------------------------------------------------------
.../simple/client/RandomBatchScanner.java | 24 ++++--
test/system/auto/simple/examples.py | 88 ++++++++++++--------
2 files changed, 68 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1fe5f7f5/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
----------------------------------------------------------------------
diff --git a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
index 7801104..604e14e 100644
--- a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
+++ b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
@@ -112,15 +112,20 @@ public class RandomBatchScanner {
* Prints a count of the number of rows mapped to false.
*
* @param expectedRows
+ * @return boolean indicating "were all the rows found?"
*/
- private static void printRowsNotFound(HashMap<Text,Boolean> expectedRows) {
+ private static boolean checkAllRowsFound(HashMap<Text,Boolean> expectedRows) {
int count = 0;
+ boolean allFound = true;
for (Entry<Text,Boolean> entry : expectedRows.entrySet())
if (!entry.getValue())
count++;
- if (count > 0)
+ if (count > 0) {
log.warn("Did not find " + count + " rows");
+ allFound = false;
+ }
+ return allFound;
}
/**
@@ -139,8 +144,9 @@ public class RandomBatchScanner {
* a random number generator
* @param tsbr
* a batch scanner
+ * @return boolean indicating "did the queries go fine?"
*/
- static void doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) {
+ static boolean doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) {
HashSet<Range> ranges = new HashSet<Range>(num);
HashMap<Text,Boolean> expectedRows = new java.util.HashMap<Text,Boolean>();
@@ -162,7 +168,7 @@ public class RandomBatchScanner {
log.info(String.format("%6.2f lookups/sec %6.2f secs\n", num / ((t2 - t1) / 1000.0), ((t2 - t1) / 1000.0)));
log.info(String.format("num results : %,d\n", receiver.count));
- printRowsNotFound(expectedRows);
+ return checkAllRowsFound(expectedRows);
}
/**
@@ -175,6 +181,7 @@ public class RandomBatchScanner {
* @throws TableNotFoundException
*/
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ boolean status = true;
String seed = null;
int index = 0;
@@ -190,7 +197,7 @@ public class RandomBatchScanner {
if (index != 11) {
System.out
.println("Usage : RandomBatchScanner [-s <seed>] <instance name> <zoo keepers> <username> <password> <table> <num> <min> <max> <expected value size> <num threads> <auths>");
- return;
+ System.exit(1);
}
String instanceName = processedArgs[0];
@@ -220,7 +227,7 @@ public class RandomBatchScanner {
r = new Random(Long.parseLong(seed));
// do one cold
- doRandomQueries(num, min, max, expectedValueSize, r, tsbr);
+ status = doRandomQueries(num, min, max, expectedValueSize, r, tsbr);
System.gc();
System.gc();
@@ -232,8 +239,11 @@ public class RandomBatchScanner {
r = new Random(Long.parseLong(seed));
// do one hot (connections already established, metadata table cached)
- doRandomQueries(num, min, max, expectedValueSize, r, tsbr);
+ status = status && doRandomQueries(num, min, max, expectedValueSize, r, tsbr);
tsbr.close();
+ if (!status) {
+ System.exit(1);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1fe5f7f5/test/system/auto/simple/examples.py
----------------------------------------------------------------------
diff --git a/test/system/auto/simple/examples.py b/test/system/auto/simple/examples.py
index 5fc9b4b..b22411a 100755
--- a/test/system/auto/simple/examples.py
+++ b/test/system/auto/simple/examples.py
@@ -41,6 +41,10 @@ class Examples(TestUtilsMixin, unittest.TestCase):
order = 21
def runExample(self, cmd):
+ self.assert_(self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd)), "example exited with error status.")
+
+ def ignoreExample(self, cmd):
+ self.comment(" Ignoring results of command.")
self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd))
def ashell(self, input):
@@ -55,6 +59,12 @@ class Examples(TestUtilsMixin, unittest.TestCase):
log.info(LINE)
def execute(self, *cmd):
+ self.assert_(self.wait(self.runOn('localhost', cmd)), "command exited with error status.")
+
+ def executeExpectFail(self, *cmd):
+ self.assert_(not self.wait(self.runOn('localhost', cmd)), "command did not exit with error status and we expected it to.")
+
+ def executeIgnoreFail(self, *cmd):
self.wait(self.runOn('localhost', cmd))
def runTest(self):
@@ -97,7 +107,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
self.ashell('createtable bloom_test\nconfig -t bloom_test -s table.bloom.enabled=true\n')
self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchWriter', '-s', '7',
INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test',
- '1000000', '0', '1000000000', '50', '2000000', '60000', '3' 'A')
+ '1000000', '0', '1000000000', '50', '2000000', '60000', '3', 'A')
self.ashell('flush -t bloom_test -w\n')
now = time.time()
self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '7',
@@ -105,7 +115,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
500, 0, 1000000000, 50, 20, 'A')
diff = time.time() - now
now = time.time()
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '8',
+ self.executeExpectFail(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '8',
INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test',
500, 0, 1000000000, 50, 20, 'A')
diff2 = time.time() - now
@@ -114,28 +124,28 @@ class Examples(TestUtilsMixin, unittest.TestCase):
self.comment("Creating a sharded index of the accumulo java files")
self.ashell('createtable shard\ncreatetable doc2term\nquit\n')
self.execute('/bin/sh', '-c',
- 'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.simple.examples.shard.Index %s %s shard %s %s 30' %
+ 'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index %s %s shard %s %s 30' %
(INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD))
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Query',
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Query',
INSTANCE_NAME, ZOOKEEPERS, 'shard', ROOT, ROOT_PASSWORD,
'foo', 'bar')
self.comment("Creating a word index of the sharded files")
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Reverse',
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Reverse',
INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD)
self.comment("Making 1000 conjunctive queries of 5 random words")
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.ContinuousQuery',
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.ContinuousQuery',
INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD, 5, 1000)
- self.execute('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt", "/tmp/failures")
+ self.executeIgnoreFail('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt", "/tmp/failures")
self.execute('hadoop', 'fs', '-mkdir', "/tmp/input")
self.comment("Starting bulk ingest example")
self.comment(" Creating some test data")
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.GenerateTestData', 0, 1000000, '/tmp/input/data')
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.SetupTable',
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData', 0, 1000000, '/tmp/input/data')
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable',
INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable')
- self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.BulkIngestExample',
+ self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample',
INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', '/tmp/input', '/tmp')
- self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.VerifyIngest',
+ self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest',
INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', 0, 1000000)
self.wait(self.runOn(self.masterHost(), [
'hadoop', 'fs', '-rmr', "/tmp/tableFile", "/tmp/nines"
@@ -146,7 +156,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.TeraSortIngest',
+ 'org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
ROWS,
10, 10,
78, 78,
@@ -160,7 +170,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.RegexExample',
+ 'org.apache.accumulo.examples.simple.mapreduce.RegexExample',
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
@@ -175,7 +185,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.RowHash',
+ 'org.apache.accumulo.examples.simple.mapreduce.RowHash',
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
@@ -188,7 +198,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.TableToFile',
+ 'org.apache.accumulo.examples.simple.mapreduce.TableToFile',
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
@@ -211,28 +221,28 @@ class Examples(TestUtilsMixin, unittest.TestCase):
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.WordCount',
+ 'org.apache.accumulo.examples.simple.mapreduce.WordCount',
INSTANCE_NAME,
ZOOKEEPERS,
'/tmp/wc',
'wctable'
]))
self.comment("Inserting data with a batch writer")
- self.runExample(['org.apache.accumulo.simple.examples.helloworld.InsertWithBatchWriter',
+ self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter',
INSTANCE_NAME,
ZOOKEEPERS,
- 'helloBatch',
ROOT,
- ROOT_PASSWORD])
+ ROOT_PASSWORD,
+ 'helloBatch'])
self.comment("Reading data")
- self.runExample(['org.apache.accumulo.simple.examples.helloworld.ReadData',
+ self.runExample(['org.apache.accumulo.examples.simple.helloworld.ReadData',
INSTANCE_NAME,
ZOOKEEPERS,
- 'helloBatch',
ROOT,
- ROOT_PASSWORD])
+ ROOT_PASSWORD,
+ 'helloBatch'])
self.comment("Running isolated scans")
- self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest',
+ self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest',
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
@@ -241,7 +251,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
100000,
'true'])
self.comment("Running scans without isolation")
- self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest',
+ self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest',
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
@@ -250,30 +260,30 @@ class Examples(TestUtilsMixin, unittest.TestCase):
100000,
'false'])
self.comment("Inserting data using a map/reduce job")
- self.runExample(['org.apache.accumulo.simple.examples.helloworld.InsertWithOutputFormat',
+ self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithOutputFormat',
INSTANCE_NAME,
ZOOKEEPERS,
- 'helloOutputFormat',
ROOT,
- ROOT_PASSWORD])
+ ROOT_PASSWORD,
+ 'helloOutputFormat'])
self.comment("Using some example constraints")
self.ashell('\n'.join([
'createtable testConstraints',
- 'config -t testConstraints -s table.constraint.1=org.apache.accumulo.simple.examples.constraints.NumericValueConstraint',
- 'config -t testConstraints -s table.constraint.2=org.apache.accumulo.simple.examples.constraints.AlphaNumKeyConstraint',
+ 'config -t testConstraints -s table.constraint.1=org.apache.accumulo.examples.simple.constraints.NumericValueConstraint',
+ 'config -t testConstraints -s table.constraint.2=org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint',
'insert r1 cf1 cq1 1111',
'insert r1 cf1 cq1 ABC',
'scan',
'quit'
]))
self.comment("Performing some row operations")
- self.runExample(['org.apache.accumulo.simple.examples.client.RowOperations',
+ self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations',
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
ROOT_PASSWORD])
self.comment("Using the batch writer")
- self.runExample(['org.apache.accumulo.simple.examples.client.SequentialBatchWriter',
+ self.runExample(['org.apache.accumulo.examples.simple.client.SequentialBatchWriter',
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
@@ -287,7 +297,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
numThreads,
visibility])
self.comment("Reading and writing some data")
- self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample',
+ self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample',
'-i', INSTANCE_NAME,
'-z', ZOOKEEPERS,
'-u', ROOT,
@@ -298,7 +308,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
'-r',
'-dbg'])
self.comment("Deleting some data")
- self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample',
+ self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample',
'-i', INSTANCE_NAME,
'-z', ZOOKEEPERS,
'-u', ROOT,
@@ -308,7 +318,9 @@ class Examples(TestUtilsMixin, unittest.TestCase):
'-d',
'-dbg'])
self.comment("Writing some random data with the batch writer")
- self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchWriter',
+ self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchWriter',
+ '-s',
+ 5,
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
@@ -322,8 +334,10 @@ class Examples(TestUtilsMixin, unittest.TestCase):
latency,
numThreads,
visibility])
- self.comment("Writing some random data with the batch writer")
- self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchScanner',
+ self.comment("Reading some random data with the batch scanner")
+ self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchScanner',
+ '-s',
+ 5,
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
@@ -336,7 +350,7 @@ class Examples(TestUtilsMixin, unittest.TestCase):
numThreads,
auths]);
self.comment("Running an example table operation (Flush)")
- self.runExample(['org.apache.accumulo.simple.examples.client.Flush',
+ self.runExample(['org.apache.accumulo.examples.simple.client.Flush',
INSTANCE_NAME,
ZOOKEEPERS,
ROOT,
[3/3] git commit: ACCUMULO-1878 fix example integration tests
Posted by ec...@apache.org.
ACCUMULO-1878 fix example integration tests
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fd652ca0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fd652ca0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fd652ca0
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: fd652ca0dd60030b6b7e665476ad2ed72574dd36
Parents: 9196910 c175bd3
Author: Eric Newton <er...@gmail.com>
Authored: Mon Nov 25 15:25:12 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 15:25:12 2013 -0500
----------------------------------------------------------------------
.../simple/client/RandomBatchScanner.java | 21 +++++++++++++------
.../mapreduce/bulk/BulkIngestExample.java | 2 +-
.../simple/mapreduce/bulk/GenerateTestData.java | 2 +-
.../accumulo/test/functional/ExamplesIT.java | 22 +++++++-------------
4 files changed, 25 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fd652ca0/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
index 2b3289e,f75f11d..5f9b975
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
@@@ -85,7 -84,7 +85,7 @@@ public class BulkIngestExample extends
int index = 0;
for (Text value : values) {
-- Key outputKey = new Key(key, new Text("foo"), new Text("" + index), timestamp);
++ Key outputKey = new Key(key, new Text("colf"), new Text(String.format("col_%07d", index)), timestamp);
index++;
Value outputValue = new Value(value.getBytes(), 0, value.getLength());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fd652ca0/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/GenerateTestData.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/GenerateTestData.java
index 9748d06,9748d06..c1a13b3
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/GenerateTestData.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/GenerateTestData.java
@@@ -45,7 -45,7 +45,7 @@@ public class GenerateTestData
PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(new Path(opts.outputFile))));
for (int i = 0; i < opts.numRows; i++) {
-- out.println(String.format("row_%08d\tvalue_%08d", i + opts.startRow, i + opts.startRow));
++ out.println(String.format("row_%010d\tvalue_%010d", i + opts.startRow, i + opts.startRow));
}
out.close();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fd652ca0/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
index 459edf3,0000000..84803f1
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
@@@ -1,291 -1,0 +1,285 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.AgeOffFilter;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.client.Flush;
+import org.apache.accumulo.examples.simple.client.RandomBatchScanner;
+import org.apache.accumulo.examples.simple.client.RandomBatchWriter;
+import org.apache.accumulo.examples.simple.client.ReadWriteExample;
+import org.apache.accumulo.examples.simple.client.RowOperations;
+import org.apache.accumulo.examples.simple.client.SequentialBatchWriter;
+import org.apache.accumulo.examples.simple.client.TraceDumpExample;
+import org.apache.accumulo.examples.simple.client.TracingExample;
+import org.apache.accumulo.examples.simple.constraints.MaxMutationSize;
+import org.apache.accumulo.examples.simple.dirlist.Ingest;
+import org.apache.accumulo.examples.simple.dirlist.QueryUtil;
+import org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter;
+import org.apache.accumulo.examples.simple.helloworld.ReadData;
+import org.apache.accumulo.examples.simple.isolation.InterferenceTest;
+import org.apache.accumulo.examples.simple.mapreduce.RegexExample;
+import org.apache.accumulo.examples.simple.mapreduce.RowHash;
+import org.apache.accumulo.examples.simple.mapreduce.TableToFile;
+import org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest;
+import org.apache.accumulo.examples.simple.mapreduce.WordCount;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable;
+import org.apache.accumulo.examples.simple.shard.ContinuousQuery;
+import org.apache.accumulo.examples.simple.shard.Index;
+import org.apache.accumulo.examples.simple.shard.Query;
+import org.apache.accumulo.examples.simple.shard.Reverse;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.MiniAccumuloCluster.LogWriter;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ExamplesIT extends ConfigurableMacIT {
+
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+
+ @Override
+ public void configure(MiniAccumuloConfig cfg) {
+ cfg.setDefaultMemory(cfg.getDefaultMemory() * 2, MemoryUnit.BYTE);
+ }
+
+ @Test(timeout = 10 * 60 * 1000)
+ public void test() throws Exception {
+ Connector c = getConnector();
+ String instance = c.getInstance().getInstanceName();
+ String keepers = c.getInstance().getZooKeepers();
+ String user = "root";
+ String passwd = ROOT_PASSWORD;
+ String visibility = "A|B";
+ String auths = "A,B";
+ BatchWriterConfig bwc = new BatchWriterConfig();
+ BatchWriter bw;
+ IteratorSetting is;
+ String dir = cluster.getConfig().getDir().getAbsolutePath();
+ FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+
+ Process trace = cluster.exec(TraceServer.class);
+ while (!c.tableOperations().exists("trace"))
+ UtilWaitThread.sleep(500);
+
+ log.info("trace example");
+ Process p = cluster.exec(TracingExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-C", "-D", "-c");
+ assertEquals(0, p.waitFor());
+ for (LogWriter writer : cluster.getLogWriters()) {
+ writer.flush();
+ }
+ String result = FunctionalTestUtils.readAll(cluster, TracingExample.class, p);
+ Pattern pattern = Pattern.compile("TraceID: ([0-9a-f]+)");
+ Matcher matcher = pattern.matcher(result);
+ int count = 0;
+ while (matcher.find()) {
+ p = cluster.exec(TraceDumpExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--traceid", matcher.group(1));
+ assertEquals(0, p.waitFor());
+ count++;
+ }
+ assertTrue(count > 0);
+ result = FunctionalTestUtils.readAll(cluster, TraceDumpExample.class, p);
+ assertTrue(result.contains("myHost@myApp"));
+ trace.destroy();
+
+ log.info("testing dirlist example (a little)");
+ c.securityOperations().changeUserAuthorizations(user, new Authorizations(auths.split(",")));
+ assertEquals(
+ 0,
+ cluster.exec(Ingest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--dirTable", "dirTable", "--indexTable", "indexTable",
+ "--dataTable", "dataTable", "--vis", visibility, "--chunkSize", 10000 + "", cluster.getConfig().getDir().getAbsolutePath()).waitFor());
+ p = cluster.exec(QueryUtil.class, "-i", instance, "-z", keepers, "-p", passwd, "-u", user, "-t", "indexTable", "--auths", auths, "--search", "--path",
+ "accumulo-site.xml");
+ assertEquals(0, p.waitFor());
+ for (LogWriter writer : cluster.getLogWriters()) {
+ writer.flush();
+ }
+ result = FunctionalTestUtils.readAll(cluster, QueryUtil.class, p);
+ System.out.println("result " + result);
+ assertTrue(result.contains("accumulo-site.xml"));
+
+ log.info("Testing ageoff filtering");
+ c.tableOperations().create("filtertest");
+ is = new IteratorSetting(10, AgeOffFilter.class);
+ AgeOffFilter.setTTL(is, 1000L);
+ c.tableOperations().attachIterator("filtertest", is);
+ bw = c.createBatchWriter("filtertest", bwc);
+ Mutation m = new Mutation("foo");
+ m.put("a", "b", "c");
+ bw.addMutation(m);
+ UtilWaitThread.sleep(1000);
+ count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> line : c.createScanner("filtertest", Authorizations.EMPTY))
+ count++;
+ assertEquals(0, count);
+
+ log.info("Testing bloom filters are fast for missing data");
+ c.tableOperations().create("bloom_test");
+ c.tableOperations().setProperty("bloom_test", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
+ assertEquals(
+ 0,
+ cluster.exec(RandomBatchWriter.class, "--seed", "7", "-i", instance, "-z", keepers, "-u", user, "-p", ROOT_PASSWORD, "--num", "100000", "--min", "0",
- "--max", "1000000000", "--size", "50", "--batchMemmory", "2M", "--batchLatency", "60s", "--batchThreads", "3", "-t", "bloom_test").waitFor());
++ "--max", "1000000000", "--size", "50", "--batchMemory", "2M", "--batchLatency", "60s", "--batchThreads", "3", "-t", "bloom_test").waitFor());
+ c.tableOperations().flush("bloom_test", null, null, true);
+ long diff = 0, diff2 = 0;
+ // try the speed test a couple times in case the system is loaded with other tests
+ for (int i = 0; i < 2; i++) {
+ long now = System.currentTimeMillis();
+ assertEquals(0, cluster.exec(RandomBatchScanner.class,"--seed", "7", "-i", instance, "-z",
+ keepers, "-u", user, "-p", ROOT_PASSWORD, "--num", "10000", "--min", "0", "--max", "1000000000", "--size", "50",
+ "--scanThreads", "4","-t", "bloom_test").waitFor());
+ diff = System.currentTimeMillis() - now;
+ now = System.currentTimeMillis();
- assertEquals(0, cluster.exec(RandomBatchScanner.class,"--seed", "8", "-i", instance, "-z",
++ assertEquals(1, cluster.exec(RandomBatchScanner.class,"--seed", "8", "-i", instance, "-z",
+ keepers, "-u", user, "-p", ROOT_PASSWORD, "--num", "10000", "--min", "0", "--max", "1000000000", "--size", "50",
+ "--scanThreads", "4","-t", "bloom_test").waitFor());
+ diff2 = System.currentTimeMillis() - now;
+ if (diff2 < diff)
+ break;
+ }
+ assertTrue(diff2 < diff);
+
+ log.info("Creating a sharded index of the accumulo java files");
+ c.tableOperations().create("shard");
+ c.tableOperations().create("doc2term");
+ bw = c.createBatchWriter("shard", bwc);
+ Index.index(30, new File(System.getProperty("user.dir") + "/src"), "\\W+", bw);
+ bw.close();
+ BatchScanner bs = c.createBatchScanner("shard", Authorizations.EMPTY, 4);
+ List<String> found = Query.query(bs, Arrays.asList("foo", "bar"));
+ bs.close();
+ // should find ourselves
+ boolean thisFile = false;
+ for (String file : found) {
+ if (file.endsWith("/ExamplesIT.java"))
+ thisFile = true;
+ }
+ assertTrue(thisFile);
+ // create a reverse index
- assertEquals(0, cluster.exec(Reverse.class, "-i", instance, "-z", keepers, "-t", "shard", "--doc2Term", "-u", "root", "-p", passwd).waitFor());
++ c.tableOperations().create("doc2Term");
++ assertEquals(0, cluster.exec(Reverse.class, "-i", instance, "-z", keepers, "--shardTable", "shard", "--doc2Term", "doc2Term", "-u", "root", "-p", passwd).waitFor());
+ // run some queries
+ assertEquals(
+ 0,
- cluster.exec(ContinuousQuery.class, "-i", instance, "-z", keepers, "-t", "shard", "--doc2Term", "-u", "root", "-p", passwd, "--term", "5", "--count",
++ cluster.exec(ContinuousQuery.class, "-i", instance, "-z", keepers, "--shardTable", "shard", "--doc2Term", "doc2Term", "-u", "root", "-p", passwd, "--terms", "5", "--count",
+ "1000").waitFor());
+
+ log.info("Testing MaxMutation constraint");
+ c.tableOperations().create("test_ingest");
+ c.tableOperations().addConstraint("test_ingest", MaxMutationSize.class.getName());
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.rows = 1;
+ opts.cols = 1000;
+ try {
+ TestIngest.ingest(c, opts, bwOpts);
+ } catch (MutationsRejectedException ex) {
+ assertEquals(1, ex.getConstraintViolationSummaries().size());
+ }
+
- log.info("Starting build ingest example");
- assertEquals(0, cluster.exec(GenerateTestData.class, "0", "10000", dir + "/tmp/input/data").waitFor());
- assertEquals(0, cluster.exec(SetupTable.class, instance, keepers, user, passwd, "bulkTable").waitFor());
- assertEquals(0, cluster.exec(BulkIngestExample.class, instance, keepers, user, passwd, "bulkTable", dir + "/tmp/input", dir + "/tmp").waitFor());
- assertEquals(0, cluster.exec(VerifyIngest.class, instance, keepers, user, passwd, "bulkTable", "0", "1000000").waitFor());
-
+ log.info("Starting bulk ingest example");
- assertEquals(0, cluster.exec(GenerateTestData.class, "0", "1000000", dir + "/tmp/input/data").waitFor());
- assertEquals(0, cluster.exec(SetupTable.class, instance, keepers, user, passwd, "bulkTable").waitFor());
- assertEquals(0, cluster.exec(BulkIngestExample.class, instance, keepers, user, passwd, "bulkTable", dir + "/tmp/input", dir + "/tmp").waitFor());
- assertEquals(0, cluster.exec(VerifyIngest.class, instance, keepers, user, passwd, "bulkTable", "0", "1000000").waitFor());
++ assertEquals(0, cluster.exec(GenerateTestData.class, "--start-row", "0", "--count", "10000", "--output", dir + "/tmp/input/data").waitFor());
++ assertEquals(0, cluster.exec(SetupTable.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "bulkTable").waitFor());
++ assertEquals(0, cluster.exec(BulkIngestExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "bulkTable", "--inputDir", dir + "/tmp/input", "--workDir", dir + "/tmp").waitFor());
+
+ log.info("Running TeraSortIngest example");
+ exec(TeraSortIngest.class, new String[] {"--count", (1000 * 1000) + "", "-nk", "10", "-xk", "10", "-nv", "10", "-xv", "10", "-t", "sorted", "-i", instance,
+ "-z", keepers, "-u", user, "-p", passwd, "--splits", "4"});
+ log.info("Running Regex example");
+ exec(RegexExample.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "sorted", "--rowRegex", ".*999.*", "--output",
+ dir + "/tmp/nines"});
+ log.info("Running RowHash example");
+ exec(RowHash.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "sorted", "--column", "c:"});
+ log.info("Running TableToFile example");
+ exec(TableToFile.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "sorted", "--output", dir + "/tmp/tableFile"});
+
+ log.info("Running word count example");
+ c.tableOperations().create("wordCount");
+ is = new IteratorSetting(10, SummingCombiner.class);
+ SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column(new Text("count"))));
+ SummingCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
+ c.tableOperations().attachIterator("wordCount", is);
+ fs.copyFromLocalFile(new Path(new Path(System.getProperty("user.dir")).getParent(), "README"), new Path(dir + "/tmp/wc/README"));
+ exec(WordCount.class, new String[] {"-i", instance, "-u", user, "-p", passwd, "-z", keepers, "--input", dir + "/tmp/wc", "-t", "wordCount"});
+
+ log.info("Inserting data with a batch writer");
+ exec(InsertWithBatchWriter.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "helloBatch"});
+ log.info("Reading data");
+ exec(ReadData.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "helloBatch"});
+ log.info("Running isolated scans");
+ exec(InterferenceTest.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "itest1", "--iterations", "100000", "--isolated"});
+ log.info("Running scans without isolation");
+ exec(InterferenceTest.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "itest2", "--iterations", "100000",});
+ log.info("Performing some row operations");
+ exec(RowOperations.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd,});
+ log.info("Using the batch writer");
+ c.tableOperations().create("test");
+ exec(SequentialBatchWriter.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "test", "--start", "0", "--num", "100000",
+ "--size", "50", "--batchMemory", "10000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility});
+
+ log.info("Reading and writing some data");
+ exec(ReadWriteExample.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", "test2", "--createtable",
+ "-c", "--debug"});
+ log.info("Deleting some data");
+ exec(ReadWriteExample.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", "test2", "-d", "--debug"});
+ log.info("Writing some data with the batch writer");
+ c.tableOperations().create("test3");
+ exec(RandomBatchWriter.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "test3", "--num", "100000", "--min", "0",
+ "--max", "99999", "--size", "100", "--batchMemory", "1000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility});
+ log.info("Reading some data with the batch scanner");
+ exec(RandomBatchScanner.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "test3", "--num", "10000", "--min", "0",
+ "--max", "99999", "--size", "100", "--scanThreads", "4", "--auths", auths});
+ log.info("Running an example table operation (Flush)");
+ exec(Flush.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "test3",});
+ assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+
+ }
+
+}
[2/3] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by ec...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
test/system/auto/simple/examples.py
In addition to merging from 1.4 also fixed a lot of issues with examples functional test that were not issues in 1.4.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c175bd31
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c175bd31
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c175bd31
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: c175bd31cfb18a081e328ccf5b172f67fcea7491
Parents: 17ac692 1fe5f7f
Author: Keith Turner <kt...@apache.org>
Authored: Mon Nov 25 13:14:32 2013 -0500
Committer: Keith Turner <kt...@apache.org>
Committed: Mon Nov 25 13:14:32 2013 -0500
----------------------------------------------------------------------
.../simple/client/RandomBatchScanner.java | 21 ++--
.../examples/simple/mapreduce/TableToFile.java | 2 +-
test/system/auto/simple/examples.py | 103 ++++++++++---------
3 files changed, 73 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
index 5194b67,0000000..2ae82b4
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
@@@ -1,225 -1,0 +1,234 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.client;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.Random;
+
+import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Internal class used to verify validity of data read.
+ */
+class CountingVerifyingReceiver {
+ private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
+
+ long count = 0;
+ int expectedValueSize = 0;
+ HashMap<Text,Boolean> expectedRows;
+
+ CountingVerifyingReceiver(HashMap<Text,Boolean> expectedRows, int expectedValueSize) {
+ this.expectedRows = expectedRows;
+ this.expectedValueSize = expectedValueSize;
+ }
+
+ public void receive(Key key, Value value) {
+
+ String row = key.getRow().toString();
+ long rowid = Integer.parseInt(row.split("_")[1]);
+
+ byte expectedValue[] = RandomBatchWriter.createValue(rowid, expectedValueSize);
+
+ if (!Arrays.equals(expectedValue, value.get())) {
+ log.error("Got unexpected value for " + key + " expected : " + new String(expectedValue) + " got : " + new String(value.get()));
+ }
+
+ if (!expectedRows.containsKey(key.getRow())) {
+ log.error("Got unexpected key " + key);
+ } else {
+ expectedRows.put(key.getRow(), true);
+ }
+
+ count++;
+ }
+}
+
+/**
+ * Simple example for reading random batches of data from Accumulo. See docs/examples/README.batch for instructions.
+ */
+public class RandomBatchScanner {
+ private static final Logger log = Logger.getLogger(CountingVerifyingReceiver.class);
+
+ /**
+ * Generate a number of ranges, each covering a single random row.
+ *
+ * @param num
+ * the number of ranges to generate
+ * @param min
+ * the minimum row that will be generated
+ * @param max
+ * the maximum row that will be generated
+ * @param r
+ * a random number generator
+ * @param ranges
+ * a set in which to store the generated ranges
+ * @param expectedRows
+ * a map in which to store the rows covered by the ranges (initially mapped to false)
+ */
+ static void generateRandomQueries(int num, long min, long max, Random r, HashSet<Range> ranges, HashMap<Text,Boolean> expectedRows) {
+ log.info(String.format("Generating %,d random queries...", num));
+ while (ranges.size() < num) {
+ long rowid = (Math.abs(r.nextLong()) % (max - min)) + min;
+
+ Text row1 = new Text(String.format("row_%010d", rowid));
+
+ Range range = new Range(new Text(row1));
+ ranges.add(range);
+ expectedRows.put(row1, false);
+ }
+
+ log.info("finished");
+ }
+
+ /**
+ * Prints a count of the number of rows mapped to false.
+ *
+ * @param expectedRows
++ * @return boolean indicating "were all the rows found?"
+ */
- private static void printRowsNotFound(HashMap<Text,Boolean> expectedRows) {
++ private static boolean checkAllRowsFound(HashMap<Text,Boolean> expectedRows) {
+ int count = 0;
++ boolean allFound = true;
+ for (Entry<Text,Boolean> entry : expectedRows.entrySet())
+ if (!entry.getValue())
+ count++;
+
- if (count > 0)
++ if (count > 0) {
+ log.warn("Did not find " + count + " rows");
++ allFound = false;
++ }
++ return allFound;
+ }
+
+ /**
+ * Generates a number of random queries, verifies that the key/value pairs returned were in the queried ranges and that the values were generated by
+ * {@link RandomBatchWriter#createValue(long, int)}. Prints information about the results.
+ *
+ * @param num
+ * the number of queries to generate
+ * @param min
+ * the min row to query
+ * @param max
+ * the max row to query
+ * @param evs
+ * the expected size of the values
+ * @param r
+ * a random number generator
+ * @param tsbr
+ * a batch scanner
++ * @return boolean indicating "did the queries go fine?"
+ */
- static void doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) {
++ static boolean doRandomQueries(int num, long min, long max, int evs, Random r, BatchScanner tsbr) {
+
+ HashSet<Range> ranges = new HashSet<Range>(num);
+ HashMap<Text,Boolean> expectedRows = new java.util.HashMap<Text,Boolean>();
+
+ generateRandomQueries(num, min, max, r, ranges, expectedRows);
+
+ tsbr.setRanges(ranges);
+
+ CountingVerifyingReceiver receiver = new CountingVerifyingReceiver(expectedRows, evs);
+
+ long t1 = System.currentTimeMillis();
+
+ for (Entry<Key,Value> entry : tsbr) {
+ receiver.receive(entry.getKey(), entry.getValue());
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ log.info(String.format("%6.2f lookups/sec %6.2f secs%n", num / ((t2 - t1) / 1000.0), ((t2 - t1) / 1000.0)));
+ log.info(String.format("num results : %,d%n", receiver.count));
+
- printRowsNotFound(expectedRows);
++ return checkAllRowsFound(expectedRows);
+ }
+
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--min", description="miniumum row that will be generated")
+ long min = 0;
+ @Parameter(names="--max", description="maximum ow that will be generated")
+ long max = 0;
+ @Parameter(names="--num", required=true, description="number of ranges to generate")
+ int num = 0;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int size = 0;
+ @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ Long seed = null;
+ }
+
+ /**
+ * Scans over a specified number of entries to Accumulo using a {@link BatchScanner}. Completes scans twice to compare times for a fresh query with those for
+ * a repeated query which has cached metadata and connections already established.
+ *
+ * @param args
+ * @throws AccumuloException
+ * @throws AccumuloSecurityException
+ * @throws TableNotFoundException
+ */
+ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ Opts opts = new Opts();
+ BatchScannerOpts bsOpts = new BatchScannerOpts();
+ opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts);
+
+ Connector connector = opts.getConnector();
+ BatchScanner batchReader = connector.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
+ batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
+
+ Random r;
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one cold
- doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
++ boolean status = doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
+
+ System.gc();
+ System.gc();
+ System.gc();
+
+ if (opts.seed == null)
+ r = new Random();
+ else
+ r = new Random(opts.seed);
+
+ // do one hot (connections already established, metadata table cached)
- doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
++ status = status && doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
+
+ batchReader.close();
++ if (!status) {
++ System.exit(1);
++ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
index 4951852,0000000..d8eedef
mode 100644,000000..100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
@@@ -1,130 -1,0 +1,130 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.examples.simple.mapreduce;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.format.DefaultFormatter;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Takes a table and outputs the specified column to a set of part files on hdfs accumulo accumulo.examples.mapreduce.TableToFile <username> <password>
+ * <tablename> <column> <hdfs-output-path>
+ */
+public class TableToFile extends Configured implements Tool {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names = "--output", description = "output directory", required = true)
+ String output;
+ @Parameter(names = "--columns", description = "columns to extract, in cf:cq{,cf:cq,...} form")
- String columns;
++ String columns = "";
+ }
+
+ /**
+ * The Mapper class that given a row number, will generate the appropriate output line.
+ */
+ public static class TTFMapper extends Mapper<Key,Value,NullWritable,Text> {
+ @Override
+ public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
+ final Key r = row;
+ final Value v = data;
+ Map.Entry<Key,Value> entry = new Map.Entry<Key,Value>() {
+ @Override
+ public Key getKey() {
+ return r;
+ }
+
+ @Override
+ public Value getValue() {
+ return v;
+ }
+
+ @Override
+ public Value setValue(Value value) {
+ return null;
+ }
+ };
+ context.write(NullWritable.get(), new Text(DefaultFormatter.formatEntry(entry, false)));
+ context.setStatus("Outputed Value");
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
+ Opts opts = new Opts();
+ opts.parseArgs(getClass().getName(), args);
+
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ HashSet<Pair<Text,Text>> columnsToFetch = new HashSet<Pair<Text,Text>>();
+ for (String col : opts.columns.split(",")) {
+ int idx = col.indexOf(":");
+ Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
+ Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
+ if (cf.getLength() > 0)
+ columnsToFetch.add(new Pair<Text,Text>(cf, cq));
+ }
+ if (!columnsToFetch.isEmpty())
+ AccumuloInputFormat.fetchColumns(job, columnsToFetch);
+
+ job.setMapperClass(TTFMapper.class);
+ job.setMapOutputKeyClass(NullWritable.class);
+ job.setMapOutputValueClass(Text.class);
+
+ job.setNumReduceTasks(0);
+
+ job.setOutputFormatClass(TextOutputFormat.class);
+ TextOutputFormat.setOutputPath(job, new Path(opts.output));
+
+ job.waitForCompletion(true);
+ return job.isSuccessful() ? 0 : 1;
+ }
+
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new TableToFile(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c175bd31/test/system/auto/simple/examples.py
----------------------------------------------------------------------
diff --cc test/system/auto/simple/examples.py
index 069148d,b22411a..3ba425e
--- a/test/system/auto/simple/examples.py
+++ b/test/system/auto/simple/examples.py
@@@ -41,11 -41,15 +41,11 @@@ class Examples(TestUtilsMixin, unittest
order = 21
def runExample(self, cmd):
- return self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd))
+ self.assert_(self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd)), "example exited with error status.")
- def ignoreExample(self, cmd):
- self.comment(" Ignoring results of command.")
- self.wait(self.runOn(self.masterHost(), [self.accumulo_sh(),] + cmd))
-
- def ashell(self, input):
+ def ashell(self, input, expected = 0):
out, err, code = self.shell(self.masterHost(), input + '\n')
- self.assert_(code == 0)
+ self.assert_(code == expected)
return out
def comment(self, description):
@@@ -58,17 -68,9 +64,16 @@@
self.wait(self.runOn('localhost', cmd))
def runTest(self):
+ examplesJar = glob.glob(ACCUMULO_HOME+'/lib/accumulo-examples-simple*.jar')[0]
-
+ self.comment("Testing MaxMutation constraint")
+ self.ashell('createtable test_ingest\n'
+ 'constraint -a org.apache.accumulo.examples.simple.constraints.MaxMutationSize\n')
+ handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.test.TestIngest', '-u', ROOT, '--rows', '1', '--start', '0', '--cols', '10000', '-p', ROOT_PASSWORD])
+ out, err = handle.communicate()
+ self.failIf(handle.returncode==0)
+ self.failUnless(err.find("MutationsRejectedException: # constraint violations : 1") >= 0, "Was able to insert a mutation larger than max size")
+
self.ashell('createtable %s\nsetauths -u %s -s A,B\nquit\n' %(table, ROOT))
- examplesJar = glob.glob(ACCUMULO_HOME+'/lib/examples-simple*.jar')[0]
-
self.comment("Testing dirlist example (a little)")
self.comment(" ingesting accumulo source")
self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.dirlist.Ingest',
@@@ -103,156 -105,167 +108,161 @@@
self.comment("Testing bloom filters are fast for missing data")
self.ashell('createtable bloom_test\nconfig -t bloom_test -s table.bloom.enabled=true\n')
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchWriter', '-s', '7',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test',
- '1000000', '0', '1000000000', '50', '2000000', '60000', '3', 'A')
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchWriter', '--seed', '7',
+ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bloom_test',
+ '--num', '1000000', '--min', '0', '--max', '1000000000', '--size', '50', '--batchMemory', '2M', '--batchLatency', '60s',
+ '--batchThreads', '3')
self.ashell('flush -t bloom_test -w\n')
now = time.time()
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '7',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test',
- 500, 0, 1000000000, 50, 20, 'A')
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '--seed', '7',
+ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bloom_test',
+ '--num', '500', '--min', '0', '--max', '1000000000', '--size', '50', '--scanThreads', 4)
diff = time.time() - now
now = time.time()
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '--seed', '8',
- self.executeExpectFail(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '-s', '8',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bloom_test',
- 500, 0, 1000000000, 50, 20, 'A')
++ self.executeExpectFail(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.client.RandomBatchScanner', '--seed', '8',
+ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bloom_test',
+ '--num', '500', '--min', '0', '--max', '1000000000', '--size', '50', '--scanThreads', 4)
diff2 = time.time() - now
self.assert_(diff2 < diff)
self.comment("Creating a sharded index of the accumulo java files")
self.ashell('createtable shard\ncreatetable doc2term\nquit\n')
self.execute('/bin/sh', '-c',
- 'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.simple.examples.shard.Index -i %s -z %s -t shard -u %s -p %s --partitions 30' %
- 'find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index %s %s shard %s %s 30' %
++ 'find server -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index -i %s -z %s -t shard -u %s -p %s --partitions 30' %
(INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD))
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Query',
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Query',
- INSTANCE_NAME, ZOOKEEPERS, 'shard', ROOT, ROOT_PASSWORD,
+ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard', '-u', ROOT, '-p', ROOT_PASSWORD,
'foo', 'bar')
self.comment("Creating a word index of the sharded files")
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.Reverse',
- '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD)
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.Reverse',
- INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD)
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '--shardTable', 'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD)
self.comment("Making 1000 conjunctive queries of 5 random words")
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.shard.ContinuousQuery',
- '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-t', 'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD, '--term', 5, '--count', 1000)
-
- self.execute('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt", "/tmp/failures")
- self.execute('hadoop', 'fs', '-mkdir', "/tmp/input")
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.shard.ContinuousQuery',
- INSTANCE_NAME, ZOOKEEPERS, 'shard', 'doc2term', ROOT, ROOT_PASSWORD, 5, 1000)
-
- self.executeIgnoreFail('hadoop', 'fs', '-rmr', "/tmp/input", "/tmp/files", "/tmp/splits.txt", "/tmp/failures")
- self.execute('hadoop', 'fs', '-mkdir', "/tmp/input")
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '--shardTable', 'shard', '--doc2Term', 'doc2term', '-u', ROOT, '-p', ROOT_PASSWORD, '--terms', 5, '--count', 1000)
++ self.executeIgnoreFail('hadoop', 'fs', '-rmr', "tmp/input", "tmp/files", "tmp/splits.txt", "tmp/failures")
++ self.execute('hadoop', 'fs', '-mkdir', "tmp/input")
self.comment("Starting bulk ingest example")
self.comment(" Creating some test data")
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.GenerateTestData', 0, 1000000, '/tmp/input/data')
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.simple.examples.mapreduce.bulk.SetupTable',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable')
- self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.BulkIngestExample',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', '/tmp/input', '/tmp')
- self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.simple.examples.mapreduce.bulk.VerifyIngest',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', 0, 1000000)
- self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData', 0, 1000000, '/tmp/input/data')
++ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData', '--start-row', 0, '--count', 1000000, '--output', 'tmp/input/data')
+ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable')
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bulkTable')
+ self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', '/tmp/input', '/tmp')
- self.execute(ACCUMULO_HOME+'/bin/tool.sh', examplesJar, 'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest',
- INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD, 'bulkTable', 0, 1000000)
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bulkTable', '--inputDir', 'tmp/input', '--workDir', 'tmp')
++ self.execute(self.accumulo_sh(), 'org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest',
++ '-i', INSTANCE_NAME, '-z', ZOOKEEPERS, '-u', ROOT, '-p', ROOT_PASSWORD, '-t', 'bulkTable', '--start-row', 0, '--count', 1000000)
self.wait(self.runOn(self.masterHost(), [
-- 'hadoop', 'fs', '-rmr', "/tmp/tableFile", "/tmp/nines"
++ 'hadoop', 'fs', '-rmr', "tmp/tableFile", "tmp/nines"
]))
self.comment("Running TeraSortIngest for a million rows")
++ self.ashell('createtable sorted\nquit\n')
# 10,000 times smaller than the real terasort
ROWS = 1000*1000
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.TeraSortIngest',
+ 'org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest',
- ROWS,
- 10, 10,
- 78, 78,
- 'sorted',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 4]))
+ '--count', ROWS,
+ '-nk', 10, '-xk', 10,
+ '-nk', 78, '-xk', 78,
+ '-t', 'sorted',
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '--splits', 4]))
self.comment("Looking for '999' in all rows")
++ self.ashell('deletetable sorted\ncreatetable sorted\nquit\n')
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.RegexExample',
+ 'org.apache.accumulo.examples.simple.mapreduce.RegexExample',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'sorted',
- '.*999.*',
- '.*',
- '.*',
- '.*',
- '/tmp/nines']))
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'sorted',
+ '--rowRegex', '.*999.*',
- '/tmp/nines']))
++ 'tmp/nines']))
self.comment("Generating hashes of each row into a new table")
++ self.ashell('deletetable sorted\ncreatetable sorted\nquit\n')
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.RowHash',
+ 'org.apache.accumulo.examples.simple.mapreduce.RowHash',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'sorted',
- ':',
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'sorted',
+ '--column', ':',
'sortedHashed',
]))
self.comment("Exporting the table to HDFS")
++ self.ashell('deletetable sorted\ncreatetable sorted\nquit\n')
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.TableToFile',
+ 'org.apache.accumulo.examples.simple.mapreduce.TableToFile',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'sortedHashed',
- ',',
- '/tmp/tableFile'
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'sorted',
- '--output', '/tmp/tableFile'
++ '--output', 'tmp/tableFile'
]))
self.comment("Running WordCount using Accumulo aggregators")
self.wait(self.runOn(self.masterHost(), [
-- 'hadoop', 'fs', '-rmr', "/tmp/wc"
++ 'hadoop', 'fs', '-rmr', "tmp/wc"
]))
self.wait(self.runOn(self.masterHost(), [
-- 'hadoop', 'fs', '-mkdir', "/tmp/wc"
++ 'hadoop', 'fs', '-mkdir', "tmp/wc"
]))
self.wait(self.runOn(self.masterHost(), [
-- 'hadoop', 'fs', '-copyFromLocal', ACCUMULO_HOME + "/README", "/tmp/wc/Accumulo.README"
++ 'hadoop', 'fs', '-copyFromLocal', ACCUMULO_HOME + "/README", "tmp/wc/Accumulo.README"
]))
- self.ashell('createtable wordCount -a count=org.apache.accumulo.core.iterators.aggregation.StringSummation\nquit\n')
+ self.ashell('createtable wordCount\nsetiter -scan -majc -minc -p 10 -n sum -class org.apache.accumulo.core.iterators.user.SummingCombiner\n\ncount\n\nSTRING\nquit\n')
self.wait(self.runOn(self.masterHost(), [
ACCUMULO_HOME+'/bin/tool.sh',
examplesJar,
- 'org.apache.accumulo.simple.examples.mapreduce.WordCount',
+ 'org.apache.accumulo.examples.simple.mapreduce.WordCount',
- INSTANCE_NAME,
- ZOOKEEPERS,
- '/tmp/wc',
- 'wctable'
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
- '--input', '/tmp/wc',
++ '-u', ROOT,
++ '-p', ROOT_PASSWORD,
++ '--input', 'tmp/wc',
+ '-t', 'wctable'
]))
self.comment("Inserting data with a batch writer")
- self.runExample(['org.apache.accumulo.simple.examples.helloworld.InsertWithBatchWriter',
+ self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'helloBatch'])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-t', 'helloBatch',
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD])
self.comment("Reading data")
- self.runExample(['org.apache.accumulo.simple.examples.helloworld.ReadData',
+ self.runExample(['org.apache.accumulo.examples.simple.helloworld.ReadData',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'helloBatch'])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-t', 'helloBatch',
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD])
self.comment("Running isolated scans")
- self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest',
+ self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'itest1',
- 100000,
- 'true'])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'itest1',
+ '--iterations', 100000,
+ '--isolated'])
self.comment("Running scans without isolation")
- self.runExample(['org.apache.accumulo.simple.examples.isolation.InterferenceTest',
+ self.runExample(['org.apache.accumulo.examples.simple.isolation.InterferenceTest',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'itest2',
- 100000,
- 'false'])
- self.comment("Inserting data using a map/reduce job")
- self.runExample(['org.apache.accumulo.examples.simple.helloworld.InsertWithOutputFormat',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- 'helloOutputFormat'])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', 'itest2',
+ '--iterations', 100000])
self.comment("Using some example constraints")
self.ashell('\n'.join([
'createtable testConstraints',
@@@ -262,83 -275,87 +272,84 @@@
'insert r1 cf1 cq1 ABC',
'scan',
'quit'
- ]))
+ ]), 1)
self.comment("Performing some row operations")
- self.runExample(['org.apache.accumulo.simple.examples.client.RowOperations',
- self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD])
++ self.runExample(['org.apache.accumulo.examples.simple.client.RowOperations',
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD ])
self.comment("Using the batch writer")
- self.runExample(['org.apache.accumulo.simple.examples.client.SequentialBatchWriter',
+ self.runExample(['org.apache.accumulo.examples.simple.client.SequentialBatchWriter',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- table,
- min,
- count,
- valueSize,
- memory,
- latency,
- numThreads,
- visibility])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', table,
+ '--start', min,
+ '--num', count,
+ '--size', valueSize,
+ '--batchMemory', memory,
+ '--batchLatency', latency,
+ '--batchThreads', numThreads,
+ '--vis', visibility])
self.comment("Reading and writing some data")
- self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample',
+ self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample',
'-i', INSTANCE_NAME,
'-z', ZOOKEEPERS,
'-u', ROOT,
'-p', ROOT_PASSWORD,
- '-s', auths,
- '-t', table,
- '-e',
- '-r',
- '-dbg'])
+ '--auths', auths,
- '-t', table,
- '--createtable',
++ '--table', table,
+ '-c',
+ '--debug'])
self.comment("Deleting some data")
- self.runExample(['org.apache.accumulo.simple.examples.client.ReadWriteExample',
+ self.runExample(['org.apache.accumulo.examples.simple.client.ReadWriteExample',
'-i', INSTANCE_NAME,
'-z', ZOOKEEPERS,
'-u', ROOT,
'-p', ROOT_PASSWORD,
-- '-s', auths,
-- '-t', table,
++ '-auths', auths,
++ '--table', table,
'-d',
- '-dbg'])
+ '--debug'])
self.comment("Writing some random data with the batch writer")
- self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchWriter',
+ self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchWriter',
- '-s',
- 5,
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- table,
- count,
- min,
- max,
- valueSize,
- memory,
- latency,
- numThreads,
- visibility])
- self.comment("Reading some random data with the batch scanner")
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', table,
++ '--seed','5',
+ '--num', count,
+ '--min', min,
+ '--max', max,
+ '--size', valueSize,
+ '--batchMemory', memory,
+ '--batchLatency', latency,
+ '--batchThreads', numThreads,
+ '--vis', visibility])
+ self.comment("Writing some random data with the batch writer")
- self.runExample(['org.apache.accumulo.simple.examples.client.RandomBatchScanner',
+ self.runExample(['org.apache.accumulo.examples.simple.client.RandomBatchScanner',
- '-s',
- 5,
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- table,
- count,
- min,
- max,
- valueSize,
- numThreads,
- auths]);
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', table,
++ '--seed','5',
+ '--num', count,
+ '--min', min,
+ '--max', max,
+ '--size', valueSize,
+ '--scanThreads', numThreads,
+ '--auths', auths]);
self.comment("Running an example table operation (Flush)")
- self.runExample(['org.apache.accumulo.simple.examples.client.Flush',
+ self.runExample(['org.apache.accumulo.examples.simple.client.Flush',
- INSTANCE_NAME,
- ZOOKEEPERS,
- ROOT,
- ROOT_PASSWORD,
- table])
+ '-i', INSTANCE_NAME,
+ '-z', ZOOKEEPERS,
+ '-u', ROOT,
+ '-p', ROOT_PASSWORD,
+ '-t', table])
self.shutdown_accumulo();