You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2012/12/19 17:25:07 UTC
svn commit: r1423923 [2/8] - in /accumulo/trunk: ./ bin/ core/
core/src/main/java/org/apache/accumulo/core/cli/
core/src/main/java/org/apache/accumulo/core/client/impl/
core/src/main/java/org/apache/accumulo/core/client/mapreduce/
core/src/main/java/or...
Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java?rev=1423923&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java (added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java Wed Dec 19 16:25:03 2012
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Random;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+
+public class BloomFilterLayerLookupTest {
+ public static void main(String[] args) throws IOException {
+ PrintStream out = System.out;
+
+ Random r = new Random();
+
+ HashSet<Integer> valsSet = new HashSet<Integer>();
+
+ for (int i = 0; i < 100000; i++) {
+ valsSet.add(r.nextInt(Integer.MAX_VALUE));
+ }
+
+ ArrayList<Integer> vals = new ArrayList<Integer>(valsSet);
+ Collections.sort(vals);
+
+ ConfigurationCopy acuconf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration());
+ acuconf.set(Property.TABLE_BLOOM_ENABLED, "true");
+ acuconf.set(Property.TABLE_BLOOM_KEY_FUNCTOR, "accumulo.core.file.keyfunctor.ColumnFamilyFunctor");
+ acuconf.set(Property.TABLE_FILE_TYPE, RFile.EXTENSION);
+ acuconf.set(Property.TABLE_BLOOM_LOAD_THRESHOLD, "1");
+ acuconf.set(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT, "1");
+
+ Configuration conf = CachedConfiguration.getInstance();
+ FileSystem fs = FileSystem.get(conf);
+
+ String suffix = FileOperations.getNewFileExtension(acuconf);
+ String fname = "/tmp/test." + suffix;
+ FileSKVWriter bmfw = FileOperations.getInstance().openWriter(fname, fs, conf, acuconf);
+
+ long t1 = System.currentTimeMillis();
+
+ bmfw.startDefaultLocalityGroup();
+
+ for (Integer i : vals) {
+ String fi = String.format("%010d", i);
+ bmfw.append(new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1")), new Value(("v" + fi).getBytes()));
+ bmfw.append(new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf2")), new Value(("v" + fi).getBytes()));
+ }
+
+ long t2 = System.currentTimeMillis();
+
+ out.printf("write rate %6.2f%n", vals.size() / ((t2 - t1) / 1000.0));
+
+ bmfw.close();
+
+ t1 = System.currentTimeMillis();
+ FileSKVIterator bmfr = FileOperations.getInstance().openReader(fname, false, fs, conf, acuconf);
+ t2 = System.currentTimeMillis();
+ out.println("Opened " + fname + " in " + (t2 - t1));
+
+ t1 = System.currentTimeMillis();
+
+ int hits = 0;
+ for (int i = 0; i < 5000; i++) {
+ int row = r.nextInt(Integer.MAX_VALUE);
+ String fi = String.format("%010d", row);
+ // bmfr.seek(new Range(new Text("r"+fi)));
+ org.apache.accumulo.core.data.Key k1 = new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1"));
+ bmfr.seek(new Range(k1, true, k1.followingKey(PartialKey.ROW_COLFAM), false), new ArrayList<ByteSequence>(), false);
+ if (valsSet.contains(row)) {
+ hits++;
+ if (!bmfr.hasTop()) {
+ out.println("ERROR " + row);
+ }
+ }
+ }
+
+ t2 = System.currentTimeMillis();
+
+ out.printf("random lookup rate : %6.2f%n", 5000 / ((t2 - t1) / 1000.0));
+ out.println("hits = " + hits);
+
+ int count = 0;
+
+ t1 = System.currentTimeMillis();
+
+ for (Integer row : valsSet) {
+ String fi = String.format("%010d", row);
+ // bmfr.seek(new Range(new Text("r"+fi)));
+
+ org.apache.accumulo.core.data.Key k1 = new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1"));
+ bmfr.seek(new Range(k1, true, k1.followingKey(PartialKey.ROW_COLFAM), false), new ArrayList<ByteSequence>(), false);
+
+ if (!bmfr.hasTop()) {
+ out.println("ERROR 2 " + row);
+ }
+
+ count++;
+
+ if (count >= 500) {
+ break;
+ }
+ }
+
+ t2 = System.currentTimeMillis();
+
+ out.printf("existant lookup rate %6.2f%n", 500 / ((t2 - t1) / 1000.0));
+ out.println("expected hits 500. Receive hits: " + count);
+ bmfr.close();
+ }
+
+}
Propchange: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java (original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/util/shell/command/FormatterCommandTest.java Wed Dec 19 16:25:03 2012
@@ -50,7 +50,7 @@ public class FormatterCommandTest {
// Keep the Shell AUDIT log off the test output
Logger.getLogger(Shell.class).setLevel(Level.WARN);
- final String[] args = new String[] {"-fake", "-u", "root", "-p", ""};
+ final String[] args = new String[] {"--fake", "-u", "root", "-p", ""};
final String[] commands = createCommands();
Modified: accumulo/trunk/docs/examples/README.batch
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.batch?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.batch (original)
+++ accumulo/trunk/docs/examples/README.batch Wed Dec 19 16:25:03 2012
@@ -40,8 +40,8 @@ Before you run this, you must ensure tha
You must also create the table, batchtest1, ahead of time. (In the shell, use "createtable batchtest1")
$ ./bin/accumulo shell -u username -e "createtable batchtest1"
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.SequentialBatchWriter instance zookeepers username password batchtest1 0 10000 50 20000000 500 20 exampleVis
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner instance zookeepers username password batchtest1 100 0 10000 50 20 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.SequentialBatchWriter -i instance -z zookeepers -u username -p password -t batchtest1 --start 0 --num 10000 --size 50 --batchMemory 20M --batchLatency 500 --batchThreads 20 --vis exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner -i instance -z zookeepers -u username -p password -t batchtest1 --num 100 --min 0 --max 10000 --size 50 --scanThreads 20 --vis exampleVis
07 11:33:11,103 [client.CountingVerifyingReceiver] INFO : Generating 100 random queries...
07 11:33:11,112 [client.CountingVerifyingReceiver] INFO : finished
07 11:33:11,260 [client.CountingVerifyingReceiver] INFO : 694.44 lookups/sec 0.14 secs
Modified: accumulo/trunk/docs/examples/README.bloom
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.bloom?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.bloom (original)
+++ accumulo/trunk/docs/examples/README.bloom Wed Dec 19 16:25:03 2012
@@ -39,7 +39,7 @@ Below 1 million random values are insert
generated rows range between 0 and 1 billion. The random number generator is
initialized with the seed 7.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter -s 7 instance zookeepers username password bloom_test 1000000 0 1000000000 50 2000000 60000 3 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter --seed 7 -i instance -z zookeepers -u username -p password -t bloom_test --num 1000000 -min 0 -max 1000000000 -valueSize 50 -batchMemory 2M -batchLatency 60s -batchThreads 3 --vis exampleVis
Below the table is flushed:
@@ -50,7 +50,7 @@ After the flush completes, 500 random qu
same seed is used to generate the queries, therefore everything is found in the
table.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner -s 7 instance zookeepers username password bloom_test 500 0 1000000000 50 20 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner --seed 7 -i instance -z zookeepers -u username -p password -t bloom_test --num 500 --min 0 --max 1000000000 --size 50 -batchThreads 20 --vis exampleVis
Generating 500 random queries...finished
96.19 lookups/sec 5.20 secs
num results : 500
@@ -62,7 +62,7 @@ Below another 500 queries are performed,
in nothing being found. In this case the lookups are much faster because of
the bloom filters.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner -s 8 instance zookeepers username password bloom_test 500 0 1000000000 50 20 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner --seed 8 -i instance -z zookeepers -u username -p password -t bloom_test --num 500 --min 0 --max 1000000000 --size 50 -batchThreads 20 -auths exampleVis
Generating 500 random queries...finished
2212.39 lookups/sec 0.23 secs
num results : 0
@@ -113,11 +113,12 @@ The commands for creating the first tabl
username@instance bloom_test1> config -t bloom_test1 -s table.compaction.major.ratio=7
username@instance bloom_test1> exit
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter -s 7 instance zookeepers username password bloom_test1 1000000 0 1000000000 50 2000000 60000 3 exampleVis
+ $ ARGS="-i instance -z zookeepers -u username -p password -t bloom_test1 --num 1000000 --min 0 --max 1000000000 --size 50 --batchMemory 2M --batchLatency 60s --batchThreads 3 --auths exampleVis"
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter --seed 7 $ARGS
$ ./bin/accumulo shell -u username -p password -e 'flush -t bloom_test1 -w'
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter -s 8 instance zookeepers username password bloom_test1 1000000 0 1000000000 50 2000000 60000 3 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter --seed 8 $ARGS
$ ./bin/accumulo shell -u username -p password -e 'flush -t bloom_test1 -w'
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter -s 9 instance zookeepers username password bloom_test1 1000000 0 1000000000 50 2000000 60000 3 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter --seed 9 $ARGS
$ ./bin/accumulo shell -u username -p password -e 'flush -t bloom_test1 -w'
The commands for creating the second table with bloom filers are below.
@@ -136,18 +137,19 @@ The commands for creating the second tab
username@instance bloom_test2> config -t bloom_test2 -s table.bloom.enabled=true
username@instance bloom_test2> exit
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter -s 7 instance zookeepers username password bloom_test2 1000000 0 1000000000 50 2000000 60000 3 exampleVis
+ $ ARGS="-i instance -z zookeepers -u username -p password -t bloom_test2 --num 1000000 --min 0 --max 1000000000 --size 50 --batchMemory 2M --batchLatency 60s --batchThreads 3 --auths exampleVis"
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter --seed 7 $ARGS
$ ./bin/accumulo shell -u username -p password -e 'flush -t bloom_test2 -w'
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter -s 8 instance zookeepers username password bloom_test2 1000000 0 1000000000 50 2000000 60000 3 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter --seed 8 $ARGS
$ ./bin/accumulo shell -u username -p password -e 'flush -t bloom_test2 -w'
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter -s 9 instance zookeepers username password bloom_test2 1000000 0 1000000000 50 2000000 60000 3 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchWriter --seed 9 $ARGS
$ ./bin/accumulo shell -u username -p password -e 'flush -t bloom_test2 -w'
Below 500 lookups are done against the table without bloom filters using random
NG seed 7. Even though only one map file will likely contain entries for this
seed, all map files will be interrogated.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner -s 7 instance zookeepers username password bloom_test1 500 0 1000000000 50 20 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner --seed 7 -i instance -z zookeepers -u username -p password -t bloom_test1 --num 500 --min 0 --max 1000000000 --size 50 --scanThreads 20 --auths exampleVis
Generating 500 random queries...finished
35.09 lookups/sec 14.25 secs
num results : 500
@@ -159,7 +161,7 @@ Below the same lookups are done against
lookups were 2.86 times faster because only one map file was used, even though three
map files existed.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner -s 7 instance zookeepers username password bloom_test2 500 0 1000000000 50 20 exampleVis
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.client.RandomBatchScanner --seed 7 -i instance -z zookeepers -u username -p password -t bloom_test2 --num 500 --min 0 --max 1000000000 --size 50 -scanThreads 20 --auths exampleVis
Generating 500 random queries...finished
99.03 lookups/sec 5.05 secs
num results : 500
Modified: accumulo/trunk/docs/examples/README.bulkIngest
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.bulkIngest?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.bulkIngest (original)
+++ accumulo/trunk/docs/examples/README.bulkIngest Wed Dec 19 16:25:03 2012
@@ -25,10 +25,10 @@ accumulo. Then we verify the 1000 rows
first two arguments to all of the commands except for GenerateTestData are the
accumulo instance name, and a comma-separated list of zookeepers.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable instance zookeepers username password test_bulk row_00000333 row_00000666
- $ ./bin/accumulo org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData 0 1000 bulk/test_1.txt
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable -i instance -z zookeepers -u username -p password -t test_bulk row_00000333 row_00000666
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData --start-row 0 --count 1000 --output bulk/test_1.txt
- $ ./bin/tool.sh lib/examples-simple-*[^cs].jar org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample instance zookeepers username password test_bulk bulk tmp/bulkWork
- $ ./bin/accumulo org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest instance zookeepers username password test_bulk 0 1000
+ $ ./bin/tool.sh lib/examples-simple-*[^cs].jar org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample -i instance -z zookeepers -u username -p password -t test_bulk bulk tmp/bulkWork
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest -i instance -z zookeepers -u username -p password -t test_bulk --start-row 0 --count 1000
For a high level discussion of bulk ingest, see the docs dir.
Modified: accumulo/trunk/docs/examples/README.dirlist
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.dirlist?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.dirlist (original)
+++ accumulo/trunk/docs/examples/README.dirlist Wed Dec 19 16:25:03 2012
@@ -31,7 +31,7 @@ This example shows how to use Accumulo t
To begin, ingest some data with Ingest.java.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.Ingest instance zookeepers username password dirTable indexTable dataTable exampleVis 100000 /local/username/workspace
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.Ingest -i instance -z zookeepers -u username -p password --vis exampleVis --chunkSize 100000 /local/username/workspace
This may take some time if there are large files in the /local/username/workspace directory. If you use 0 instead of 100000 on the command line, the ingest will run much faster, but it will not put any file data into Accumulo (the dataTable will be empty).
Note that running this example will create tables dirTable, indexTable, and dataTable in Accumulo that you should delete when you have completed the example.
@@ -43,20 +43,20 @@ To browse the data ingested, use Viewer.
then run the Viewer:
- $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.Viewer instance zookeepers username password dirTable dataTable exampleVis /local/username/workspace
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.Viewer -i instance -z zookeepers -u username -p password -t dirTable --dataTable dataTable --auths exampleVis --path /local/username/workspace
To list the contents of specific directories, use QueryUtil.java.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil instance zookeepers username password dirTable exampleVis /local/username
- $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil instance zookeepers username password dirTable exampleVis /local/username/workspace
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil -i instance -z zookeepers -u username -p password -t dirTable --auths exampleVis --path /local/username
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil -i instance -z zookeepers -u username -p password -t dirTable --auths exampleVis --path /local/username/workspace
To perform searches on file or directory names, also use QueryUtil.java. Search terms must contain no more than one wild card and cannot contain "/".
*Note* these queries run on the _indexTable_ table instead of the dirTable table.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil instance zookeepers username password indexTable exampleVis filename -search
- $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil instance zookeepers username password indexTable exampleVis 'filename*' -search
- $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil instance zookeepers username password indexTable exampleVis '*jar' -search
- $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil instance zookeepers username password indexTable exampleVis filename*jar -search
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil -i instance -z zookeepers -u username -p password -t indexTable --auths exampleVis --path filename --search
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil -i instance -z zookeepers -u username -p password -t indexTable --auths exampleVis --path 'filename*' --search
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil -i instance -z zookeepers -u username -p password -t indexTable --auths exampleVis --path '*jar' --search
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.dirlist.QueryUtil -i instance -z zookeepers -u username -p password -t indexTable --auths exampleVis iipath 'filename*jar' --search
To count the number of direct children (directories and files) and descendants (children and children's descendants, directories and files), run the FileCount over the dirTable table.
The results are written back to the same table. FileCount reads from and writes to Accumulo. This requires scan authorizations for the read and a visibility for the data written.
Modified: accumulo/trunk/docs/examples/README.filedata
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.filedata?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.filedata (original)
+++ accumulo/trunk/docs/examples/README.filedata Wed Dec 19 16:25:03 2012
@@ -32,7 +32,7 @@ This example is coupled with the dirlist
If you haven't already run the README.dirlist example, ingest a file with FileDataIngest.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.filedata.FileDataIngest instance zookeepers username password dataTable exampleVis 1000 $ACCUMULO_HOME/README
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.filedata.FileDataIngest -i instance -z zookeepers -u username -p password -t dataTable --auths exampleVis --chunk 1000 $ACCUMULO_HOME/README
Open the accumulo shell and look at the data. The row is the MD5 hash of the file, which you can verify by running a command such as 'md5sum' on the file.
@@ -40,7 +40,7 @@ Open the accumulo shell and look at the
Run the CharacterHistogram MapReduce to add some information about the file.
- $ bin/tool.sh lib/examples-simple*[^cs].jar org.apache.accumulo.examples.simple.filedata.CharacterHistogram instance zookeepers username password dataTable exampleVis exampleVis
+ $ bin/tool.sh lib/examples-simple*[^cs].jar org.apache.accumulo.examples.simple.filedata.CharacterHistogram -i instance -z zookeepers -u username -p password -t dataTable --auths exampleVis --vis exampleVis
Scan again to see the histogram stored in the 'info' column family.
Modified: accumulo/trunk/docs/examples/README.helloworld
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.helloworld?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.helloworld (original)
+++ accumulo/trunk/docs/examples/README.helloworld Wed Dec 19 16:25:03 2012
@@ -32,11 +32,11 @@ Create a table called 'hellotable':
Launch a Java program that inserts data with a BatchWriter:
- $ ./bin/accumulo org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter instance zookeepers username password hellotable
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter -i instance -z zookeepers -u username -p password -t hellotable
Alternatively, the same data can be inserted using MapReduce writers:
- $ ./bin/accumulo org.apache.accumulo.examples.simple.helloworld.InsertWithOutputFormat instance zookeepers username password hellotable
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.helloworld.InsertWithOutputFormat -i instance -z zookeepers -u username -p password -t hellotable
On the accumulo status page at the URL below (where 'master' is replaced with the name or IP of your accumulo master), you should see 50K entries
@@ -49,4 +49,4 @@ To view the entries, use the shell to sc
You can also use a Java class to scan the table:
- $ ./bin/accumulo org.apache.accumulo.examples.simple.helloworld.ReadData instance zookeepers username password hellotable row_0 row_1001
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.helloworld.ReadData -i instance -z zookeepers -u username -p password -t hellotable --startKey row_0 --endKey row_1001
Modified: accumulo/trunk/docs/examples/README.isolation
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.isolation?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.isolation (original)
+++ accumulo/trunk/docs/examples/README.isolation Wed Dec 19 16:25:03 2012
@@ -31,7 +31,7 @@ reading the row at the same time a mutat
Below, Interference Test is run without isolation enabled for 5000 iterations
and it reports problems.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.isolation.InterferenceTest instance zookeepers username password isotest 5000 false
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.isolation.InterferenceTest -i instance -z zookeepers -u username -p password -t isotest --iterations 5000
ERROR Columns in row 053 had multiple values [53, 4553]
ERROR Columns in row 061 had multiple values [561, 61]
ERROR Columns in row 070 had multiple values [570, 1070]
@@ -44,7 +44,7 @@ and it reports problems.
Below, Interference Test is run with isolation enabled for 5000 iterations and
it reports no problems.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.isolation.InterferenceTest instance zookeepers username password isotest 5000 true
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.isolation.InterferenceTest -i instance -z zookeepers -u username -p password -t isotest --iterations 5000 --isolated
finished
Modified: accumulo/trunk/docs/examples/README.mapred
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.mapred?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.mapred (original)
+++ accumulo/trunk/docs/examples/README.mapred Wed Dec 19 16:25:03 2012
@@ -50,7 +50,7 @@ for the column family count.
After creating the table, run the word count map reduce job.
- $ bin/tool.sh lib/examples-simple*[^cs].jar org.apache.accumulo.examples.simple.mapreduce.WordCount instance zookeepers /user/username/wc wordCount -u username -p password
+ $ bin/tool.sh lib/examples-simple*[^cs].jar org.apache.accumulo.examples.simple.mapreduce.WordCount -i instance -z zookeepers --input /user/username/wc wordCount -u username -p password
11/02/07 18:20:11 INFO input.FileInputFormat: Total input paths to process : 1
11/02/07 18:20:12 INFO mapred.JobClient: Running job: job_201102071740_0003
Modified: accumulo/trunk/docs/examples/README.maxmutation
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.maxmutation?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.maxmutation (original)
+++ accumulo/trunk/docs/examples/README.maxmutation Wed Dec 19 16:25:03 2012
@@ -19,9 +19,9 @@ Notice: Licensed to the Apache Softwa
This an example of how to limit the size of mutations that will be accepted into
a table. Under the default configuration, accumulo does not provide a limitation
on the size of mutations that can be ingested. Poorly behaved writers might
-inadvertently create mutations so large, that they cause the tablet servers
-or logger processes to run out of memory. A simple contraint can be added to
-a table to reject very large mutations.
+inadvertently create mutations so large, that they cause the tablet servers to
+run out of memory. A simple contraint can be added to a table to reject very
+large mutations.
$ ./bin/accumulo shell -u username -p password
@@ -42,6 +42,6 @@ Now the table will reject any mutation t
working memory of the tablet server. The following command attempts to ingest
a single row with 10000 columns, which exceeds the memory limit:
- $ ./bin/accumulo org.apache.accumulo.server.test.TestIngest 1 0 10000
+ $ ./bin/accumulo org.apache.accumulo.server.test.TestIngest -i instance -z zookeepers -u username -p password --rows 1 --cols 10000
ERROR : Constraint violates : ConstraintViolationSummary(constrainClass:org.apache.accumulo.examples.simple.constraints.MaxMutationSize, violationCode:0, violationDescription:mutation exceeded maximum size of 188160, numberOfViolatingMutations:1)
Modified: accumulo/trunk/docs/examples/README.shard
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.shard?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.shard (original)
+++ accumulo/trunk/docs/examples/README.shard Wed Dec 19 16:25:03 2012
@@ -32,12 +32,12 @@ To run these example programs, create tw
After creating the tables, index some files. The following command indexes all of the java files in the Accumulo source code.
$ cd /local/username/workspace/accumulo/
- $ find src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index instance zookeepers shard username password 30
+ $ find core/src server/src -name "*.java" | xargs ./bin/accumulo org.apache.accumulo.examples.simple.shard.Index -i instance -z zookeepers -t shard -u username -p password --partitions 30
The following command queries the index to find all files containing 'foo' and 'bar'.
$ cd $ACCUMULO_HOME
- $ ./bin/accumulo org.apache.accumulo.examples.simple.shard.Query instance zookeepers shard username password foo bar
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.shard.Query -i instance -z zookeepers -t shard -u username -p password foo bar
/local/username/workspace/accumulo/src/core/src/test/java/accumulo/core/security/ColumnVisibilityTest.java
/local/username/workspace/accumulo/src/core/src/test/java/accumulo/core/client/mock/MockConnectorTest.java
/local/username/workspace/accumulo/src/core/src/test/java/accumulo/core/security/VisibilityEvaluatorTest.java
@@ -52,11 +52,12 @@ The following command queries the index
In order to run ContinuousQuery, we need to run Reverse.java to populate doc2term.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.shard.Reverse instance zookeepers shard doc2term username password
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.shard.Reverse -i instance -z zookeepers --shardTable shard --doc2Term doc2term -u username -p password
-Below ContinuousQuery is run using 5 terms. So it selects 5 random terms from each document, then it continually randomly selects one set of 5 terms and queries. It prints the number of matching documents and the time in seconds.
+Below ContinuousQuery is run using 5 terms. So it selects 5 random terms from each document, then it continually
+randomly selects one set of 5 terms and queries. It prints the number of matching documents and the time in seconds.
- $ ./bin/accumulo org.apache.accumulo.examples.simple.shard.ContinuousQuery instance zookeepers shard doc2term username password 5
+ $ ./bin/accumulo org.apache.accumulo.examples.simple.shard.ContinuousQuery -i instance -z zookeepers --shardTable shard --doc2Term doc2term -u username -p password --terms 5
[public, core, class, binarycomparable, b] 2 0.081
[wordtodelete, unindexdocument, doctablename, putdelete, insert] 1 0.041
[import, columnvisibilityinterpreterfactory, illegalstateexception, cv, columnvisibility] 1 0.049
Modified: accumulo/trunk/docs/examples/README.visibility
URL: http://svn.apache.org/viewvc/accumulo/trunk/docs/examples/README.visibility?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/docs/examples/README.visibility (original)
+++ accumulo/trunk/docs/examples/README.visibility Wed Dec 19 16:25:03 2012
@@ -128,4 +128,4 @@ It can be set with the following constra
row f3:q3 [(apple&carrot)|broccoli|spinach] v3
row f4:q4 [spinach|broccoli] v4
username@instance vistest>
-
\ No newline at end of file
+
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/Flush.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/Flush.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/Flush.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/Flush.java Wed Dec 19 16:25:03 2012
@@ -16,8 +16,8 @@
*/
package org.apache.accumulo.examples.simple.client;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
/**
* Simple example for using tableOperations() (like create, delete, flush, etc).
@@ -25,21 +25,11 @@ import org.apache.accumulo.core.client.Z
public class Flush {
public static void main(String[] args) {
- if (args.length != 5) {
- System.err.println("Usage: accumulo accumulo.examples.client.Flush <instance name> <zoo keepers> <username> <password> <tableName>");
- return;
- }
- String instanceName = args[0];
- String zooKeepers = args[1];
- String user = args[2];
- String password = args[3];
- String table = args[4];
-
- Connector connector;
+ ClientOnRequiredTable opts = new ClientOnRequiredTable();
+ opts.parseArgs(Flush.class.getName(), args);
try {
- ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
- connector = instance.getConnector(user, password.getBytes());
- connector.tableOperations().flush(table, null, null, true);
+ Connector connector = opts.getConnector();
+ connector.tableOperations().flush(opts.tableName, null, null, true);
} catch (Exception e) {
throw new RuntimeException(e);
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java Wed Dec 19 16:25:03 2012
@@ -22,19 +22,20 @@ import java.util.HashSet;
import java.util.Map.Entry;
import java.util.Random;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
+import com.beust.jcommander.Parameter;
+
/**
* Internal class used to verify validity of data read.
*/
@@ -165,6 +166,19 @@ public class RandomBatchScanner {
printRowsNotFound(expectedRows);
}
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--min", description="miniumum row that will be generated")
+ long min = 0;
+ @Parameter(names="--max", description="maximum ow that will be generated")
+ long max = 0;
+ @Parameter(names="--num", required=true, description="number of ranges to generate")
+ int num = 0;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int size = 0;
+ @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ Long seed = null;
+ }
+
/**
* Scans over a specified number of entries to Accumulo using a {@link BatchScanner}. Completes scans twice to compare times for a fresh query with those for
* a repeated query which has cached metadata and connections already established.
@@ -175,64 +189,32 @@ public class RandomBatchScanner {
* @throws TableNotFoundException
*/
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- String seed = null;
-
- int index = 0;
- String processedArgs[] = new String[11];
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("-s")) {
- seed = args[++i];
- } else {
- processedArgs[index++] = args[i];
- }
- }
-
- if (index != 11) {
- System.out
- .println("Usage : RandomBatchScanner [-s <seed>] <instance name> <zoo keepers> <username> <password> <table> <num> <min> <max> <expected value size> <num threads> <auths>");
- return;
- }
+ Opts opts = new Opts();
+ opts.parseArgs(RandomBatchScanner.class.getName(), args);
- String instanceName = processedArgs[0];
- String zooKeepers = processedArgs[1];
- String user = processedArgs[2];
- byte[] pass = processedArgs[3].getBytes();
- String table = processedArgs[4];
- int num = Integer.parseInt(processedArgs[5]);
- long min = Long.parseLong(processedArgs[6]);
- long max = Long.parseLong(processedArgs[7]);
- int expectedValueSize = Integer.parseInt(processedArgs[8]);
- int numThreads = Integer.parseInt(processedArgs[9]);
- String auths = processedArgs[10];
-
- // Uncomment the following lines for detailed debugging info
- // Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
- // logger.setLevel(Level.TRACE);
-
- ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
- Connector connector = instance.getConnector(user, pass);
- BatchScanner tsbr = connector.createBatchScanner(table, new Authorizations(auths.split(",")), numThreads);
+ Connector connector = opts.getConnector();
+ BatchScanner tsbr = connector.createBatchScanner(opts.tableName, opts.auths, opts.scanThreads);
Random r;
- if (seed == null)
+ if (opts.seed == null)
r = new Random();
else
- r = new Random(Long.parseLong(seed));
+ r = new Random(opts.seed);
// do one cold
- doRandomQueries(num, min, max, expectedValueSize, r, tsbr);
+ doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, tsbr);
System.gc();
System.gc();
System.gc();
- if (seed == null)
+ if (opts.seed == null)
r = new Random();
else
- r = new Random(Long.parseLong(seed));
+ r = new Random(opts.seed);
// do one hot (connections already established, metadata table cached)
- doRandomQueries(num, min, max, expectedValueSize, r, tsbr);
+ doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, tsbr);
tsbr.close();
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java Wed Dec 19 16:25:03 2012
@@ -18,22 +18,22 @@ package org.apache.accumulo.examples.sim
import java.util.HashSet;
import java.util.Random;
-import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.io.Text;
+import com.beust.jcommander.Parameter;
+
/**
* Simple example for writing random data to Accumulo. See docs/examples/README.batch for instructions.
*
@@ -90,6 +90,21 @@ public class RandomBatchWriter {
return m;
}
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--num", required=true)
+ int num = 0;
+ @Parameter(names="--min")
+ long min = 0;
+ @Parameter(names="--max")
+ long max = Long.MAX_VALUE;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int size = 0;
+ @Parameter(names="--vis", converter=VisibilityConverter.class)
+ ColumnVisibility visiblity = new ColumnVisibility("");
+ @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ Long seed = null;
+ }
+
/**
* Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
*
@@ -98,63 +113,27 @@ public class RandomBatchWriter {
* @throws TableNotFoundException
*/
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
-
- String seed = null;
-
- int index = 0;
- String processedArgs[] = new String[13];
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("-s")) {
- seed = args[++i];
- } else {
- processedArgs[index++] = args[i];
- }
- }
-
- if (index != 13) {
- System.out
- .println("Usage : RandomBatchWriter [-s <seed>] <instance name> <zoo keepers> <username> <password> <table> <num> <min> <max> <value size> <max memory> <max latency> <num threads> <visibility>");
- return;
- }
-
- String instanceName = processedArgs[0];
- String zooKeepers = processedArgs[1];
- String user = processedArgs[2];
- byte[] pass = processedArgs[3].getBytes();
- String table = processedArgs[4];
- int num = Integer.parseInt(processedArgs[5]);
- long min = Long.parseLong(processedArgs[6]);
- long max = Long.parseLong(processedArgs[7]);
- int valueSize = Integer.parseInt(processedArgs[8]);
- long maxMemory = Long.parseLong(processedArgs[9]);
- long maxLatency = Long.parseLong(processedArgs[10]) == 0 ? Long.MAX_VALUE : Long.parseLong(processedArgs[10]);
- int numThreads = Integer.parseInt(processedArgs[11]);
- String visiblity = processedArgs[12];
-
- // Uncomment the following lines for detailed debugging info
- // Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
- // logger.setLevel(Level.TRACE);
+ Opts opts = new Opts();
+ opts.parseArgs(RandomBatchWriter.class.getName(), args);
Random r;
- if (seed == null)
+ if (opts.seed == null)
r = new Random();
else {
- r = new Random(Long.parseLong(seed));
+ r = new Random(opts.seed);
}
- ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
- Connector connector = instance.getConnector(user, pass);
- BatchWriter bw = connector.createBatchWriter(table, new BatchWriterConfig().setMaxMemory(maxMemory).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS)
- .setMaxWriteThreads(numThreads));
+ Connector connector = opts.getConnector();
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, opts.getBatchWriterConfig());
// reuse the ColumnVisibility object to improve performance
- ColumnVisibility cv = new ColumnVisibility(visiblity);
+ ColumnVisibility cv = opts.visiblity;
- for (int i = 0; i < num; i++) {
+ for (int i = 0; i < opts.num; i++) {
- long rowid = (Math.abs(r.nextLong()) % (max - min)) + min;
+ long rowid = (Math.abs(r.nextLong()) % (opts.max - opts.min)) + opts.min;
- Mutation m = createMutation(rowid, valueSize, cv);
+ Mutation m = createMutation(rowid, opts.size, cv);
bw.addMutation(m);
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java Wed Dec 19 16:25:03 2012
@@ -20,184 +20,118 @@ import java.util.Map.Entry;
import java.util.SortedSet;
import java.util.TreeSet;
-import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
public class ReadWriteExample {
// defaults
- private static final String DEFAULT_INSTANCE_NAME = "test";
- private static final String DEFAULT_ZOOKEEPERS = "localhost";
private static final String DEFAULT_AUTHS = "LEVEL1,GROUP1";
private static final String DEFAULT_TABLE_NAME = "test";
- // options
- private Option instanceOpt = new Option("i", "instance", true, "instance name");
- private Option zooKeepersOpt = new Option("z", "zooKeepers", true, "zoo keepers");
-
- private Option usernameOpt = new Option("u", "user", true, "user name");
- private Option passwordOpt = new Option("p", "password", true, "password");
- private Option scanAuthsOpt = new Option("s", "scanauths", true, "comma-separated scan authorizations");
-
- private Option tableNameOpt = new Option("t", "table", true, "table name");
- private Option createtableOpt = new Option("C", "createtable", false, "create table before doing anything");
- private Option deletetableOpt = new Option("D", "deletetable", false, "delete table when finished");
-
- private Option createEntriesOpt = new Option("e", "create", false, "create entries before any deletes");
- private Option deleteEntriesOpt = new Option("d", "delete", false, "delete entries after any creates");
- private Option readEntriesOpt = new Option("r", "read", false, "read entries after any creates/deletes");
-
- private Option debugOpt = new Option("dbg", "debug", false, "enable debugging");
-
- private Options opts;
- private CommandLine cl;
private Connector conn;
+ class Opts extends ClientOnDefaultTable {
+ @Parameter(names={"-C", "--createtable"}, description="create table before doing anything")
+ boolean createtable = false;
+ @Parameter(names={"-D", "--deletetable"}, description="delete table when finished")
+ boolean deletetable = false;
+ @Parameter(names={"-c", "--create"}, description="create entries before any deletes")
+ boolean createEntries = false;
+ @Parameter(names={"-r", "--read"}, description="read entries after any creates/deletes")
+ boolean readEntries = false;
+ @Parameter(names={"-d", "--delete"}, description="delete entries after any creates")
+ boolean deleteEntries = false;
+
+ public Opts() {
+ super(DEFAULT_TABLE_NAME);
+ auths = new Authorizations(DEFAULT_AUTHS.split(","));
+ }
+ }
+
// hidden constructor
private ReadWriteExample() {}
// setup
- private void configure(String[] args) throws ParseException, AccumuloException, AccumuloSecurityException {
- usernameOpt.setRequired(true);
- passwordOpt.setRequired(true);
- opts = new Options();
- addOptions(instanceOpt, zooKeepersOpt, usernameOpt, passwordOpt, scanAuthsOpt, tableNameOpt, createtableOpt, deletetableOpt, createEntriesOpt,
- deleteEntriesOpt, readEntriesOpt, debugOpt);
-
- // parse command line
- cl = new BasicParser().parse(opts, args);
- if (cl.getArgs().length != 0)
- throw new ParseException("unrecognized options " + cl.getArgList());
+ private Opts configure(String[] args) throws ParseException, AccumuloException, AccumuloSecurityException {
+ Opts opts = new Opts();
+ opts.parseArgs(ReadWriteExample.class.getName(), args);
- // optionally enable debugging
- if (hasOpt(debugOpt))
- Logger.getLogger(Constants.CORE_PACKAGE_NAME).setLevel(Level.TRACE);
-
- Instance inst = new ZooKeeperInstance(getOpt(instanceOpt, DEFAULT_INSTANCE_NAME), getOpt(zooKeepersOpt, DEFAULT_ZOOKEEPERS));
- conn = inst.getConnector(getRequiredOpt(usernameOpt), getRequiredOpt(passwordOpt).getBytes());
- }
-
- // for setup
- private void addOptions(Option... addOpts) {
- for (Option opt : addOpts)
- opts.addOption(opt);
- }
-
- // for checking for and getting options
- private boolean hasOpt(Option opt) {
- return cl.hasOption(opt.getOpt());
- }
-
- private String getRequiredOpt(Option opt) {
- return getOpt(opt, null);
+ conn = opts.getConnector();
+ return opts;
}
- private String getOpt(Option opt, String defaultValue) {
- return cl.getOptionValue(opt.getOpt(), defaultValue);
- }
-
- // for usage
- private void printHelp() {
- HelpFormatter hf = new HelpFormatter();
- instanceOpt.setArgName("name");
- zooKeepersOpt.setArgName("hosts");
- usernameOpt.setArgName("user");
- passwordOpt.setArgName("pass");
- scanAuthsOpt.setArgName("scanauths");
- tableNameOpt.setArgName("name");
- hf.printHelp("accumulo accumulo.examples.client.ReadWriteExample", opts, true);
- }
-
- private void execute() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, MutationsRejectedException {
+ private void execute(Opts opts) throws Exception {
// create table
- if (hasOpt(createtableOpt)) {
+ if (opts.createtable) {
SortedSet<Text> partitionKeys = new TreeSet<Text>();
for (int i = Byte.MIN_VALUE; i < Byte.MAX_VALUE; i++)
partitionKeys.add(new Text(new byte[] {(byte) i}));
- conn.tableOperations().create(getOpt(tableNameOpt, DEFAULT_TABLE_NAME));
- conn.tableOperations().addSplits(getOpt(tableNameOpt, DEFAULT_TABLE_NAME), partitionKeys);
+ conn.tableOperations().create(opts.getTableName());
+ conn.tableOperations().addSplits(opts.getTableName(), partitionKeys);
}
-
- // create entries
- if (hasOpt(createEntriesOpt))
- createEntries(false);
-
- // delete entries
- if (hasOpt(deleteEntriesOpt))
- createEntries(true);
+
+ // send mutations
+ createEntries(opts);
// read entries
- if (hasOpt(readEntriesOpt)) {
+ if (opts.readEntries) {
// Note that the user needs to have the authorizations for the specified scan authorizations
// by an administrator first
- Authorizations scanauths = new Authorizations(getOpt(scanAuthsOpt, DEFAULT_AUTHS).split(","));
-
- Scanner scanner = conn.createScanner(getOpt(tableNameOpt, DEFAULT_TABLE_NAME), scanauths);
+ Scanner scanner = conn.createScanner(opts.getTableName(), opts.auths);
+ scanner.setBatchSize(opts.scanBatchSize);
for (Entry<Key,Value> entry : scanner)
System.out.println(entry.getKey().toString() + " -> " + entry.getValue().toString());
}
// delete table
- if (hasOpt(deletetableOpt))
- conn.tableOperations().delete(getOpt(tableNameOpt, DEFAULT_TABLE_NAME));
+ if (opts.deletetable)
+ conn.tableOperations().delete(opts.getTableName());
}
- private void createEntries(boolean delete) throws AccumuloException, TableNotFoundException, MutationsRejectedException {
- BatchWriter writer = conn.createBatchWriter(getOpt(tableNameOpt, DEFAULT_TABLE_NAME), new BatchWriterConfig());
- ColumnVisibility cv = new ColumnVisibility(DEFAULT_AUTHS.replace(',', '|'));
-
- Text cf = new Text("datatypes");
- Text cq = new Text("xml");
- byte[] row = {'h', 'e', 'l', 'l', 'o', '\0'};
- byte[] value = {'w', 'o', 'r', 'l', 'd', '\0'};
-
- for (int i = 0; i < 10; i++) {
- row[row.length - 1] = (byte) i;
- Mutation m = new Mutation(new Text(row));
- if (delete) {
- m.putDelete(cf, cq, cv);
- } else {
- value[value.length - 1] = (byte) i;
- m.put(cf, cq, cv, new Value(value));
+ private void createEntries(Opts opts) throws Exception {
+ if (opts.createEntries || opts.deleteEntries) {
+ BatchWriter writer = conn.createBatchWriter(opts.getTableName(), new BatchWriterConfig());
+ ColumnVisibility cv = new ColumnVisibility(opts.auths.toString().replace(',', '|'));
+
+ Text cf = new Text("datatypes");
+ Text cq = new Text("xml");
+ byte[] row = {'h', 'e', 'l', 'l', 'o', '\0'};
+ byte[] value = {'w', 'o', 'r', 'l', 'd', '\0'};
+
+ for (int i = 0; i < 10; i++) {
+ row[row.length - 1] = (byte) i;
+ Mutation m = new Mutation(new Text(row));
+ if (opts.deleteEntries) {
+ m.putDelete(cf, cq, cv);
+ }
+ if (opts.createEntries) {
+ value[value.length - 1] = (byte) i;
+ m.put(cf, cq, cv, new Value(value));
+ }
+ writer.addMutation(m);
}
- writer.addMutation(m);
+ writer.close();
}
- writer.close();
}
public static void main(String[] args) throws Exception {
ReadWriteExample rwe = new ReadWriteExample();
- try {
- rwe.configure(args);
- rwe.execute();
- } catch (ParseException e) {
- System.err.println(e.getMessage());
- rwe.printHelp();
- System.exit(1);
- }
+ Opts opts = rwe.configure(args);
+ rwe.execute(opts);
}
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RowOperations.java Wed Dec 19 16:25:03 2012
@@ -20,16 +20,15 @@ import java.io.IOException;
import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.ClientOpts;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
@@ -50,13 +49,12 @@ public class RowOperations {
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
MutationsRejectedException {
- if (args.length != 4) {
- log.error("Usage: <instance name> <zoo keepers> <username> <password>");
- return;
- }
+
+ ClientOpts opts = new ClientOpts();
+ opts.parseArgs(RowOperations.class.getName(), args);
// First the setup work
- connector = new ZooKeeperInstance(args[0], args[1]).getConnector(args[2], args[3].getBytes());
+ connector = opts.getConnector();
// lets create an example table
connector.tableOperations().create(table);
@@ -94,7 +92,7 @@ public class RowOperations {
mut3.put(new Text("column"), col4, System.currentTimeMillis(), new Value("This is the value for this key".getBytes()));
// Now we'll make a Batch Writer
- bw = connector.createBatchWriter(table, new BatchWriterConfig());
+ bw = connector.createBatchWriter(table, opts.getBatchWriterConfig());
// And add the mutations
bw.addMutation(mut1);
@@ -105,9 +103,9 @@ public class RowOperations {
bw.flush();
// Now lets look at the rows
- Scanner rowThree = getRow(new Text("row3"));
- Scanner rowTwo = getRow(new Text("row2"));
- Scanner rowOne = getRow(new Text("row1"));
+ Scanner rowThree = getRow(opts, new Text("row3"));
+ Scanner rowTwo = getRow(opts, new Text("row2"));
+ Scanner rowOne = getRow(opts, new Text("row1"));
// And print them
log.info("This is everything");
@@ -117,13 +115,13 @@ public class RowOperations {
System.out.flush();
// Now lets delete rowTwo with the iterator
- rowTwo = getRow(new Text("row2"));
+ rowTwo = getRow(opts, new Text("row2"));
deleteRow(rowTwo);
// Now lets look at the rows again
- rowThree = getRow(new Text("row3"));
- rowTwo = getRow(new Text("row2"));
- rowOne = getRow(new Text("row1"));
+ rowThree = getRow(opts, new Text("row3"));
+ rowTwo = getRow(opts, new Text("row2"));
+ rowOne = getRow(opts, new Text("row1"));
// And print them
log.info("This is row1 and row3");
@@ -135,12 +133,12 @@ public class RowOperations {
// Should only see the two rows
// Now lets delete rowOne without passing in the iterator
- deleteRow(row1);
+ deleteRow(opts, row1);
// Now lets look at the rows one last time
- rowThree = getRow(new Text("row3"));
- rowTwo = getRow(new Text("row2"));
- rowOne = getRow(new Text("row1"));
+ rowThree = getRow(opts, new Text("row3"));
+ rowTwo = getRow(opts, new Text("row2"));
+ rowOne = getRow(opts, new Text("row1"));
// And print them
log.info("This is just row3");
@@ -164,14 +162,15 @@ public class RowOperations {
/**
* Deletes a row given a text object
+ * @param opts
*
* @param row
* @throws TableNotFoundException
* @throws AccumuloSecurityException
* @throws AccumuloException
*/
- private static void deleteRow(Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- deleteRow(getRow(row));
+ private static void deleteRow(ClientOpts opts, Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ deleteRow(getRow(opts, row));
}
/**
@@ -206,6 +205,7 @@ public class RowOperations {
/**
* Gets a scanner over one row
+ * @param opts
*
* @param row
* @return
@@ -214,9 +214,10 @@ public class RowOperations {
* @throws AccumuloException
* @throws IOException
*/
- private static Scanner getRow(Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ private static Scanner getRow(ClientOpts opts, Text row) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
// Create a scanner
Scanner scanner = connector.createScanner(table, Constants.NO_AUTHS);
+ scanner.setBatchSize(opts.scanBatchSize);
// Say start key is the one with key of row
// and end key is the one that immediately follows the row
scanner.setRange(new Range(row));
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/SequentialBatchWriter.java Wed Dec 19 16:25:03 2012
@@ -16,23 +16,34 @@
*/
package org.apache.accumulo.examples.simple.client;
-import java.util.concurrent.TimeUnit;
-
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.security.ColumnVisibility;
+import com.beust.jcommander.Parameter;
+
/**
* Simple example for writing random data in sequential order to Accumulo. See docs/examples/README.batch for instructions.
*/
public class SequentialBatchWriter {
+
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--start")
+ long start = 0;
+ @Parameter(names="--num", required=true)
+ long num = 0;
+ @Parameter(names="--size", required=true, description="size of the value to write")
+ int valueSize = 0;
+ @Parameter(names="--vis", converter=VisibilityConverter.class)
+ ColumnVisibility vis = new ColumnVisibility();
+ }
+
/**
* Writes a specified number of entries to Accumulo using a {@link BatchWriter}. The rows of the entries will be sequential starting at a specified number.
* The column families will be "foo" and column qualifiers will be "1". The values will be random byte arrays of a specified size.
@@ -43,41 +54,15 @@ public class SequentialBatchWriter {
* @throws MutationsRejectedException
*/
public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException {
- if (args.length != 12) {
- System.out
- .println("Usage : SequentialBatchWriter <instance name> <zoo keepers> <username> <password> <table> <start> <num> <value size> <max memory> <max latency> <num threads> <visibility>");
- return;
- }
-
- String instanceName = args[0];
- String zooKeepers = args[1];
- String user = args[2];
- byte[] pass = args[3].getBytes();
- String table = args[4];
- long start = Long.parseLong(args[5]);
- long num = Long.parseLong(args[6]);
- int valueSize = Integer.parseInt(args[7]);
- long maxMemory = Long.parseLong(args[8]);
- long maxLatency = Long.parseLong(args[9]) == 0 ? Long.MAX_VALUE : Long.parseLong(args[9]);
- int numThreads = Integer.parseInt(args[10]);
- String visibility = args[11];
-
- // Uncomment the following lines for detailed debugging info
- // Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
- // logger.setLevel(Level.TRACE);
-
- ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
- Connector connector = instance.getConnector(user, pass);
- BatchWriter bw = connector.createBatchWriter(table, new BatchWriterConfig().setMaxMemory(maxMemory).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS)
- .setMaxWriteThreads(numThreads));
-
- long end = start + num;
+ Opts opts = new Opts();
+ opts.parseArgs(SequentialBatchWriter.class.getName(), args);
+ Connector connector = opts.getConnector();
+ BatchWriter bw = connector.createBatchWriter(opts.tableName, opts.getBatchWriterConfig());
- // reuse ColumnVisibility object for better performance
- ColumnVisibility cv = new ColumnVisibility(visibility);
+ long end = opts.start + opts.num;
- for (long i = start; i < end; i++) {
- Mutation m = RandomBatchWriter.createMutation(i, valueSize, cv);
+ for (long i = opts.start; i < end; i++) {
+ Mutation m = RandomBatchWriter.createMutation(i, opts.valueSize, opts.vis);
bw.addMutation(m);
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/FileCount.java Wed Dec 19 16:25:03 2012
@@ -19,21 +19,19 @@ package org.apache.accumulo.examples.sim
import java.util.Iterator;
import java.util.Map.Entry;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.io.Text;
+import com.beust.jcommander.Parameter;
+
/**
* Computes recursive counts over file system information and stores them back into the same Accumulo table. See docs/examples/README.dirlist for instructions.
*/
@@ -42,10 +40,7 @@ public class FileCount {
private int entriesScanned;
private int inserts;
- private Connector conn;
- private Authorizations auths;
- private ColumnVisibility colvis;
- private String table;
+ private Opts opts;
private static class CountValue {
int dirCount = 0;
@@ -172,7 +167,7 @@ public class FileCount {
private Mutation createMutation(int depth, String dir, CountValue countVal) {
Mutation m = new Mutation(String.format("%03d%s", depth, dir));
- m.put(QueryUtil.DIR_COLF, QueryUtil.COUNTS_COLQ, colvis, countVal.toValue());
+ m.put(QueryUtil.DIR_COLF, QueryUtil.COUNTS_COLQ, opts.visibility, countVal.toValue());
return m;
}
@@ -202,7 +197,6 @@ public class FileCount {
} else if (!currentDir.equals(dir)) {
batchWriter.addMutation(createMutation(depth - 1, currentDir, countVal));
inserts++;
-
currentDir = dir;
countVal.clear();
}
@@ -216,7 +210,7 @@ public class FileCount {
// in this case the higher depth will not insert anything if the
// dir has no children, so insert something here
Mutation m = new Mutation(key.getRow());
- m.put(QueryUtil.DIR_COLF, QueryUtil.COUNTS_COLQ, colvis, tmpCount.toValue());
+ m.put(QueryUtil.DIR_COLF, QueryUtil.COUNTS_COLQ, opts.visibility, tmpCount.toValue());
batchWriter.addMutation(m);
inserts++;
}
@@ -235,20 +229,8 @@ public class FileCount {
}
}
- FileCount(String instance, String zookeepers, String user, String password, String table, String auths, String colvis, boolean mock) throws Exception {
- Instance inst;
- if (mock) {
- inst = new MockInstance(instance);
- } else {
- inst = new ZooKeeperInstance(instance, zookeepers);
- }
- this.conn = inst.getConnector(user, password);
- if (auths.length() > 0)
- this.auths = new Authorizations(auths.split(","));
- else
- this.auths = new Authorizations();
- this.colvis = new ColumnVisibility(colvis);
- this.table = table;
+ FileCount(Opts opts) throws Exception {
+ this.opts = opts;
}
public void run() throws Exception {
@@ -256,8 +238,10 @@ public class FileCount {
entriesScanned = 0;
inserts = 0;
- Scanner scanner = conn.createScanner(table, auths);
- BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ Connector conn = opts.getConnector();
+ Scanner scanner = conn.createScanner(opts.tableName, opts.auths);
+ scanner.setBatchSize(opts.scanBatchSize);
+ BatchWriter bw = conn.createBatchWriter(opts.tableName, opts.getBatchWriterConfig());
long t1 = System.currentTimeMillis();
@@ -282,14 +266,17 @@ public class FileCount {
System.out.printf("Counts inserted : %,d %n", inserts);
}
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--vis", description="use a given visibility for the new counts", converter=VisibilityConverter.class)
+ ColumnVisibility visibility = new ColumnVisibility();
+ }
+
public static void main(String[] args) throws Exception {
- if (args.length != 7) {
- System.out.println("usage: " + FileCount.class.getSimpleName() + " <instance> <zookeepers> <user> <pass> <table> <auths> <colvis>");
- System.exit(1);
- }
-
- FileCount fileCount = new FileCount(args[0], args[1], args[2], args[3], args[4], args[5], args[6], false);
-
+ Opts opts = new Opts();
+ String programName = FileCount.class.getName();
+ opts.parseArgs(programName, args);
+
+ FileCount fileCount = new FileCount(opts);
fileCount.run();
}
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Ingest.java Wed Dec 19 16:25:03 2012
@@ -18,12 +18,13 @@ package org.apache.accumulo.examples.sim
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.accumulo.core.cli.ClientOpts;
import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.LongCombiner;
@@ -33,6 +34,8 @@ import org.apache.accumulo.examples.simp
import org.apache.accumulo.examples.simple.filedata.FileDataIngest;
import org.apache.hadoop.io.Text;
+import com.beust.jcommander.Parameter;
+
/**
* Recursively lists the files and directories under a given path, ingests their names and file info into one Accumulo table, indexes the file names in a
* separate table, and the file data into a third table. See docs/examples/README.dirlist for instructions.
@@ -116,50 +119,52 @@ public class Ingest {
}
}
+ static class Opts extends ClientOpts {
+ @Parameter(names="--dirTable", description="a table to hold the directory information")
+ String nameTable = "dirTable";
+ @Parameter(names="--indexTable", description="an index over the ingested data")
+ String indexTable = "indexTable";
+ @Parameter(names="--dataTable", description="the file data, chunked into parts")
+ String dataTable = "dataTable";
+ @Parameter(names="--vis", description="the visibility to mark the data", converter=VisibilityConverter.class)
+ ColumnVisibility visibility = new ColumnVisibility();
+ @Parameter(names="--chunkSize", description="the size of chunks when breaking down files")
+ int chunkSize = 100000;
+ @Parameter(description="<dir> { <dir> ... }")
+ List<String> directories = new ArrayList<String>();
+ }
+
+
public static void main(String[] args) throws Exception {
- if (args.length < 10) {
- System.out.println("usage: " + Ingest.class.getSimpleName()
- + " <instance> <zoo> <user> <pass> <dir table> <index table> <data table> <visibility> <data chunk size> <dir>{ <dir>}");
- System.exit(1);
- }
+ Opts opts = new Opts();
+ opts.parseArgs(Ingest.class.getName(), args);
- String instance = args[0];
- String zooKeepers = args[1];
- String user = args[2];
- String pass = args[3];
- String nameTable = args[4];
- String indexTable = args[5];
- String dataTable = args[6];
- ColumnVisibility colvis = new ColumnVisibility(args[7]);
- int chunkSize = Integer.parseInt(args[8]);
-
- Connector conn = new ZooKeeperInstance(instance, zooKeepers).getConnector(user, pass.getBytes());
- if (!conn.tableOperations().exists(nameTable))
- conn.tableOperations().create(nameTable);
- if (!conn.tableOperations().exists(indexTable))
- conn.tableOperations().create(indexTable);
- if (!conn.tableOperations().exists(dataTable)) {
- conn.tableOperations().create(dataTable);
- conn.tableOperations().attachIterator(dataTable, new IteratorSetting(1, ChunkCombiner.class));
- }
-
- BatchWriter dirBW = conn.createBatchWriter(nameTable, new BatchWriterConfig());
- BatchWriter indexBW = conn.createBatchWriter(indexTable, new BatchWriterConfig());
- BatchWriter dataBW = conn.createBatchWriter(dataTable, new BatchWriterConfig());
- FileDataIngest fdi = new FileDataIngest(chunkSize, colvis);
- for (int i = 9; i < args.length; i++) {
- recurse(new File(args[i]), colvis, dirBW, indexBW, fdi, dataBW);
+ Connector conn = opts.getConnector();
+ if (!conn.tableOperations().exists(opts.nameTable))
+ conn.tableOperations().create(opts.nameTable);
+ if (!conn.tableOperations().exists(opts.indexTable))
+ conn.tableOperations().create(opts.indexTable);
+ if (!conn.tableOperations().exists(opts.dataTable)) {
+ conn.tableOperations().create(opts.dataTable);
+ conn.tableOperations().attachIterator(opts.dataTable, new IteratorSetting(1, ChunkCombiner.class));
+ }
+
+ BatchWriter dirBW = conn.createBatchWriter(opts.nameTable, opts.getBatchWriterConfig());
+ BatchWriter indexBW = conn.createBatchWriter(opts.indexTable, opts.getBatchWriterConfig());
+ BatchWriter dataBW = conn.createBatchWriter(opts.dataTable, opts.getBatchWriterConfig());
+ FileDataIngest fdi = new FileDataIngest(opts.chunkSize, opts.visibility);
+ for (String dir : opts.directories) {
+ recurse(new File(dir), opts.visibility, dirBW, indexBW, fdi, dataBW);
// fill in parent directory info
- String file = args[i];
int slashIndex = -1;
- while ((slashIndex = file.lastIndexOf("/")) > 0) {
- file = file.substring(0, slashIndex);
- ingest(new File(file), colvis, dirBW, indexBW, fdi, dataBW);
+ while ((slashIndex = dir.lastIndexOf("/")) > 0) {
+ dir = dir.substring(0, slashIndex);
+ ingest(new File(dir), opts.visibility, dirBW, indexBW, fdi, dataBW);
}
}
- ingest(new File("/"), colvis, dirBW, indexBW, fdi, dataBW);
-
+ ingest(new File("/"), opts.visibility, dirBW, indexBW, fdi, dataBW);
+
dirBW.close();
indexBW.close();
dataBW.close();
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java Wed Dec 19 16:25:03 2012
@@ -20,13 +20,13 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
@@ -34,6 +34,8 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.security.Authorizations;
import org.apache.hadoop.io.Text;
+import com.beust.jcommander.Parameter;
+
/**
* Provides utility methods for getting the info for a file, listing the contents of a directory, and performing single wild card searches on file or directory
* names. See docs/examples/README.dirlist for instructions.
@@ -48,12 +50,11 @@ public class QueryUtil {
public static final Text INDEX_COLF = new Text("i");
public static final Text COUNTS_COLQ = new Text("counts");
- public QueryUtil(String instanceName, String zooKeepers, String user, String password, String tableName, Authorizations auths) throws AccumuloException,
+ public QueryUtil(Opts opts) throws AccumuloException,
AccumuloSecurityException {
- ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
- conn = instance.getConnector(user, password.getBytes());
- this.tableName = tableName;
- this.auths = auths;
+ conn = opts.getConnector();
+ this.tableName = opts.tableName;
+ this.auths = opts.auths;
}
/**
@@ -252,6 +253,13 @@ public class QueryUtil {
return scanner;
}
+ public static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--path", description="the directory to list")
+ String path = "/";
+ @Parameter(names="--search", description="find a file or directorys with the given name")
+ boolean search = false;
+ }
+
/**
* Lists the contents of a directory using the directory table, or searches for file or directory names (if the -search flag is included).
*
@@ -259,18 +267,17 @@ public class QueryUtil {
* @throws Exception
*/
public static void main(String[] args) throws Exception {
- if (args.length != 7 && (args.length != 8 || !args[7].equals("-search"))) {
- System.out.println("usage: " + QueryUtil.class.getSimpleName() + " <instance> <zookeepers> <user> <pass> <table> <auths> <path> [-search]");
- System.exit(1);
- }
- QueryUtil q = new QueryUtil(args[0], args[1], args[2], args[3], args[4], new Authorizations(args[5].split(",")));
- if (args.length == 8) {
- for (Entry<Key,Value> e : q.singleWildCardSearch(args[6])) {
+ Opts opts = new Opts();
+ opts.parseArgs(QueryUtil.class.getName(), args);
+ QueryUtil q = new QueryUtil(opts);
+ if (opts.search) {
+ for (Entry<Key,Value> e : q.singleWildCardSearch(opts.path)) {
System.out.println(e.getKey().getColumnQualifier());
}
- }
- for (Entry<String,Map<String,String>> e : q.getDirList(args[6]).entrySet()) {
- System.out.println(e);
+ } else {
+ for (Entry<String,Map<String,String>> e : q.getDirList(opts.path).entrySet()) {
+ System.out.println(e);
+ }
}
}
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java Wed Dec 19 16:25:03 2012
@@ -36,10 +36,11 @@ import javax.swing.tree.DefaultTreeModel
import javax.swing.tree.TreePath;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.examples.simple.filedata.FileDataQuery;
import org.apache.log4j.Logger;
+import com.beust.jcommander.Parameter;
+
/**
* Provides a GUI for browsing the file system information stored in Accumulo. See docs/examples/README.dirlist for instructions.
*/
@@ -96,14 +97,14 @@ public class Viewer extends JFrame imple
}
}
- public Viewer(String instanceName, String zooKeepers, String user, String password, String tableName, String dataTableName, Authorizations auths, String path)
+ public Viewer(Opts opts)
throws Exception {
super("File Viewer");
setSize(1000, 800);
setDefaultCloseOperation(EXIT_ON_CLOSE);
- q = new QueryUtil(instanceName, zooKeepers, user, password, tableName, auths);
- fdq = new FileDataQuery(instanceName, zooKeepers, user, password, dataTableName, auths);
- this.topPath = path;
+ q = new QueryUtil(opts);
+ fdq = new FileDataQuery(opts.instance, opts.zookeepers, opts.user, opts.getPassword(), opts.dataTable, opts.auths);
+ this.topPath = opts.path;
}
public void populate(DefaultMutableTreeNode node) throws TableNotFoundException {
@@ -200,15 +201,16 @@ public class Viewer extends JFrame imple
}
}
+ static class Opts extends QueryUtil.Opts {
+ @Parameter(names="--dataTable")
+ String dataTable = "dataTable";
+ }
+
public static void main(String[] args) throws Exception {
- if (args.length != 7 && args.length != 8) {
- System.out.println("usage: " + Viewer.class.getSimpleName() + " <instance> <zoo> <user> <pass> <table> <datatable> <auths> [rootpath]");
- System.exit(1);
- }
- String rootpath = "/";
- if (args.length == 8)
- rootpath = args[7];
- Viewer v = new Viewer(args[0], args[1], args[2], args[3], args[4], args[5], new Authorizations(args[6].split(",")), rootpath);
+ Opts opts = new Opts();
+ opts.parseArgs(Viewer.class.getName(), args);
+
+ Viewer v = new Viewer(opts);
v.init();
v.setVisible(true);
}
Modified: accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/CharacterHistogram.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/CharacterHistogram.java?rev=1423923&r1=1423922&r2=1423923&view=diff
==============================================================================
--- accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/CharacterHistogram.java (original)
+++ accumulo/trunk/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/CharacterHistogram.java Wed Dec 19 16:25:03 2012
@@ -22,12 +22,12 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map.Entry;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.SummingArrayCombiner;
-import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.hadoop.conf.Configured;
@@ -37,6 +37,8 @@ import org.apache.hadoop.mapreduce.Mappe
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import com.beust.jcommander.Parameter;
+
/**
* A MapReduce that computes a histogram of byte frequency for each file and stores the histogram alongside the file data. The {@link ChunkInputFormat} is used
* to read the file data from Accumulo. See docs/examples/README.filedata for instructions.
@@ -72,15 +74,23 @@ public class CharacterHistogram extends
}
}
+ static class Opts extends ClientOnRequiredTable {
+ @Parameter(names="--vis")
+ String visibilities = "";
+ }
+
+
@Override
public int run(String[] args) throws Exception {
Job job = new Job(getConf(), this.getClass().getSimpleName());
job.setJarByClass(this.getClass());
-
+
+ Opts opts = new Opts();
+ opts.parseArgs(CharacterHistogram.class.getName(), args);
+
job.setInputFormatClass(ChunkInputFormat.class);
- ChunkInputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]);
- ChunkInputFormat.setInputInfo(job.getConfiguration(), args[2], args[3].getBytes(), args[4], new Authorizations(args[5].split(",")));
- job.getConfiguration().set(VIS, args[6]);
+ opts.setAccumuloConfigs(job);
+ job.getConfiguration().set(VIS, opts.visibilities.toString());
job.setMapperClass(HistMapper.class);
job.setMapOutputKeyClass(Text.class);
@@ -89,8 +99,6 @@ public class CharacterHistogram extends
job.setNumReduceTasks(0);
job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]);
- AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[2], args[3].getBytes(), false, args[4]);
job.waitForCompletion(true);
return job.isSuccessful() ? 0 : 1;