You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2011/11/01 17:37:23 UTC
svn commit: r1196106 - in /incubator/accumulo/branches/1.3: bin/tool.sh
docs/examples/README.bloom docs/examples/README.bulkIngest
docs/examples/README.dirlist docs/examples/README.filter
docs/examples/README.mapred src/assemble/build.sh
Author: ecn
Date: Tue Nov 1 16:37:23 2011
New Revision: 1196106
URL: http://svn.apache.org/viewvc?rev=1196106&view=rev
Log:
ACCUMULO-86: cleanup all examples, especially map-reduce and jar file pattern matching
Modified:
incubator/accumulo/branches/1.3/bin/tool.sh
incubator/accumulo/branches/1.3/docs/examples/README.bloom
incubator/accumulo/branches/1.3/docs/examples/README.bulkIngest
incubator/accumulo/branches/1.3/docs/examples/README.dirlist
incubator/accumulo/branches/1.3/docs/examples/README.filter
incubator/accumulo/branches/1.3/docs/examples/README.mapred
incubator/accumulo/branches/1.3/src/assemble/build.sh
Modified: incubator/accumulo/branches/1.3/bin/tool.sh
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.3/bin/tool.sh?rev=1196106&r1=1196105&r2=1196106&view=diff
==============================================================================
--- incubator/accumulo/branches/1.3/bin/tool.sh (original)
+++ incubator/accumulo/branches/1.3/bin/tool.sh Tue Nov 1 16:37:23 2011
@@ -31,10 +31,10 @@ fi
LIB=$ACCUMULO_HOME/lib
-ZOOKEEPER_CMD='ls -1 $ZOOKEEPER_HOME/*.jar | grep -e "zookeeper-[^-]*.jar$"'
-CORE_CMD='ls -1 $LIB/accumulo-core-*.jar | grep -v -e ".*javadoc.jar$"'
-THRIFT_CMD='ls -1 $LIB/libthrift-*.jar | grep -v -e ".*javadoc.jar$"'
-CLOUDTRACE_CMD='ls -1 $LIB/cloudtrace-*.jar | grep -v -e ".*javadoc.jar$"'
+ZOOKEEPER_CMD='ls -1 $ZOOKEEPER_HOME/zookeeper-[0-9]*.jar'
+CORE_CMD='ls -1 $LIB/accumulo-core-*[^c].jar'
+THRIFT_CMD='ls -1 $LIB/libthrift-*[^c].jar'
+CLOUDTRACE_CMD='ls -1 $LIB/cloudtrace-*[^c].jar'
if [ `eval $ZOOKEEPER_CMD | wc -l` != "1" ] ; then
echo "Not exactly one zookeeper jar in $ZOOKEEPER_HOME"
Modified: incubator/accumulo/branches/1.3/docs/examples/README.bloom
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.3/docs/examples/README.bloom?rev=1196106&r1=1196105&r2=1196106&view=diff
==============================================================================
--- incubator/accumulo/branches/1.3/docs/examples/README.bloom (original)
+++ incubator/accumulo/branches/1.3/docs/examples/README.bloom Tue Nov 1 16:37:23 2011
@@ -42,13 +42,16 @@ initialized with the seed 7.
$ ./bin/accumulo org.apache.accumulo.examples.client.RandomBatchWriter -s 7 instance zookeepers username password bloom_test 1000000 0 1000000000 50 2000000 60000 3 exampleVis
Below the table is flushed, look at the monitor page and wait for the flush to
-complete.
+complete.
$ ./bin/accumulo shell -u username -p password
username@instance> flush -t bloom_test
Flush of table bloom_test initiated...
username@instance> exit
+The flush will be finished when there are no entries in memory and the
+number of minor compactions goes to zero. Refresh the page to see changes to the table.
+
After the flush completes, 500 random queries are done against the table. The
same seed is used to generate the queries, therefore everything is found in the
table.
Modified: incubator/accumulo/branches/1.3/docs/examples/README.bulkIngest
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.3/docs/examples/README.bulkIngest?rev=1196106&r1=1196105&r2=1196106&view=diff
==============================================================================
--- incubator/accumulo/branches/1.3/docs/examples/README.bulkIngest (original)
+++ incubator/accumulo/branches/1.3/docs/examples/README.bulkIngest Tue Nov 1 16:37:23 2011
@@ -28,7 +28,7 @@ accumulo instance name, and a comma-sepa
$ ./bin/accumulo org.apache.accumulo.examples.mapreduce.bulk.SetupTable instance zookeepers username password test_bulk row_00000333 row_00000666
$ ./bin/accumulo org.apache.accumulo.examples.mapreduce.bulk.GenerateTestData 0 1000 bulk/test_1.txt
- $ ./bin/tool.sh lib/accumulo-examples-*.jar org.apache.accumulo.examples.mapreduce.bulk.BulkIngestExample instance zookeepers username password test_bulk bulk tmp/bulkWork
+ $ ./bin/tool.sh lib/accumulo-examples-*[^c].jar org.apache.accumulo.examples.mapreduce.bulk.BulkIngestExample instance zookeepers username password test_bulk bulk tmp/bulkWork
$ ./bin/accumulo org.apache.accumulo.examples.mapreduce.bulk.VerifyIngest instance zookeepers username password test_bulk 0 1000
For a high level discussion of bulk ingest, see the docs dir.
Modified: incubator/accumulo/branches/1.3/docs/examples/README.dirlist
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.3/docs/examples/README.dirlist?rev=1196106&r1=1196105&r2=1196106&view=diff
==============================================================================
--- incubator/accumulo/branches/1.3/docs/examples/README.dirlist (original)
+++ incubator/accumulo/branches/1.3/docs/examples/README.dirlist Tue Nov 1 16:37:23 2011
@@ -42,16 +42,16 @@ To list the contents of specific directo
$ ./bin/accumulo org.apache.accumulo.examples.dirlist.QueryUtil instance zookeepers username password direxample exampleVis /local/user1/workspace
To perform searches on file or directory names, also use QueryUtil.java. Search terms must contain no more than one wild card and cannot contain "/".
-Note these queries run on the dirindex table instead of the direxample table.
+*Note* these queries run on the _dirindex_ table instead of the direxample table.
$ ./bin/accumulo org.apache.accumulo.examples.dirlist.QueryUtil instance zookeepers username password dirindex exampleVis filename -search
- $ ./bin/accumulo org.apache.accumulo.examples.dirlist.QueryUtil instance zookeepers username password dirindex exampleVis filename* -search
- $ ./bin/accumulo org.apache.accumulo.examples.dirlist.QueryUtil instance zookeepers username password dirindex exampleVis *jar -search
+ $ ./bin/accumulo org.apache.accumulo.examples.dirlist.QueryUtil instance zookeepers username password dirindex exampleVis 'filename*' -search
+ $ ./bin/accumulo org.apache.accumulo.examples.dirlist.QueryUtil instance zookeepers username password dirindex exampleVis '*jar' -search
$ ./bin/accumulo org.apache.accumulo.examples.dirlist.QueryUtil instance zookeepers username password dirindex exampleVis filename*jar -search
To count the number of direct children (directories and files) and descendants (children and children's descendents, directories and files), run the FileCountMR over the direxample table.
The results can be written back to the same table.
- $ ./bin/tool.sh lib/accumulo-examples-*.jar org.apache.accumulo.examples.dirlist.FileCountMR instance zookeepers username password direxample direxample exampleVis exampleVis
+ $ ./bin/tool.sh lib/accumulo-examples-*[^c].jar org.apache.accumulo.examples.dirlist.FileCountMR instance zookeepers username password direxample direxample exampleVis exampleVis
Alternatively, you can also run FileCount.java.
Modified: incubator/accumulo/branches/1.3/docs/examples/README.filter
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.3/docs/examples/README.filter?rev=1196106&r1=1196105&r2=1196106&view=diff
==============================================================================
--- incubator/accumulo/branches/1.3/docs/examples/README.filter (original)
+++ incubator/accumulo/branches/1.3/docs/examples/README.filter Tue Nov 1 16:37:23 2011
@@ -36,7 +36,6 @@ value pair are to be delivered and false
username@instance filtertest> scan
username@instance filtertest> insert foo a b c
- insert successful
username@instance filtertest> scan
foo a:b [] c
@@ -56,6 +55,16 @@ on the "minc" and "majc" scopes you can
happen automatically as a background operation on any table that is being
actively written to, but these are the commands to force compaction:
+ username@instance filtertest> setiter -t filtertest -scan -minc -majc -p 10 -n myfilter -filter
+ FilteringIterator uses Filters to accept or reject key/value pairs
+ ----------> entering options: <filterPriorityNumber> <ageoff|regex|filterClass>
+ ----------> set org.apache.accumulo.core.iterators.FilteringIterator option (<name> <value>, hit enter to skip): 0 ageoff
+ ----------> set org.apache.accumulo.core.iterators.FilteringIterator option (<name> <value>, hit enter to skip):
+ AgeOffFilter removes entries with timestamps more than <ttl> milliseconds old
+ ----------> set org.apache.accumulo.core.iterators.filter.AgeOffFilter parameter currentTime, if set, use the given value as the absolute time in milliseconds as the current time of day:
+ ----------> set org.apache.accumulo.core.iterators.filter.AgeOffFilter parameter ttl, time to live (milliseconds): 30000
+ username@instance filtertest>
+
username@instance filtertest> flush -t filtertest
08 11:13:55,745 [shell.Shell] INFO : Flush of table filtertest initiated...
username@instance filtertest> compact -t filtertest
@@ -71,15 +80,21 @@ To see the iterator settings for a table
---------+------------------------------------------+----------------------------------------------------------
SCOPE | NAME | VALUE
---------+------------------------------------------+----------------------------------------------------------
- table | table.iterator.majc.vers................ | 20,org.apache.accumulo.core.iterators.VersioningIterator
- table | table.iterator.majc.vers.opt.maxVersions | 1
- table | table.iterator.minc.vers................ | 20,org.apache.accumulo.core.iterators.VersioningIterator
- table | table.iterator.minc.vers.opt.maxVersions | 1
- table | table.iterator.scan.myfilter............ | 10,org.apache.accumulo.core.iterators.FilteringIterator
- table | table.iterator.scan.myfilter.opt.0...... | org.apache.accumulo.core.iterators.filter.AgeOffFilter
- table | table.iterator.scan.myfilter.opt.0.ttl.. | 30000
- table | table.iterator.scan.vers................ | 20,org.apache.accumulo.core.iterators.VersioningIterator
- table | table.iterator.scan.vers.opt.maxVersions | 1
+ table | table.iterator.majc.myfilter .............. | 10,org.apache.accumulo.core.iterators.FilteringIterator
+ table | table.iterator.majc.myfilter.opt.0 ........ | org.apache.accumulo.core.iterators.filter.AgeOffFilter
+ table | table.iterator.majc.myfilter.opt.0.ttl .... | 30000
+ table | table.iterator.majc.vers .................. | 20,org.apache.accumulo.core.iterators.VersioningIterator
+ table | table.iterator.majc.vers.opt.maxVersions .. | 1
+ table | table.iterator.minc.myfilter .............. | 10,org.apache.accumulo.core.iterators.FilteringIterator
+ table | table.iterator.minc.myfilter.opt.0 ........ | org.apache.accumulo.core.iterators.filter.AgeOffFilter
+ table | table.iterator.minc.myfilter.opt.0.ttl .... | 30000
+ table | table.iterator.minc.vers .................. | 20,org.apache.accumulo.core.iterators.VersioningIterator
+ table | table.iterator.minc.vers.opt.maxVersions .. | 1
+ table | table.iterator.scan.myfilter .............. | 10,org.apache.accumulo.core.iterators.FilteringIterator
+ table | table.iterator.scan.myfilter.opt.0 ........ | org.apache.accumulo.core.iterators.filter.AgeOffFilter
+ table | table.iterator.scan.myfilter.opt.0.ttl .... | 30000
+ table | table.iterator.scan.vers .................. | 20,org.apache.accumulo.core.iterators.VersioningIterator
+ table | table.iterator.scan.vers.opt.maxVersions .. | 1
---------+------------------------------------------+----------------------------------------------------------
username@instance filtertest>
Modified: incubator/accumulo/branches/1.3/docs/examples/README.mapred
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.3/docs/examples/README.mapred?rev=1196106&r1=1196105&r2=1196106&view=diff
==============================================================================
--- incubator/accumulo/branches/1.3/docs/examples/README.mapred (original)
+++ incubator/accumulo/branches/1.3/docs/examples/README.mapred Tue Nov 1 16:37:23 2011
@@ -17,7 +17,7 @@ Notice: Licensed to the Apache Softwa
under the License.
This example uses mapreduce and accumulo to compute word counts for a set of
-documents. This is accomplished using a map only map reduce job and a
+documents. This is accomplished using a map-only mapreduce job and a
accumulo table with aggregators.
To run this example you will need a directory in HDFS containing text files.
@@ -44,7 +44,7 @@ for the column family count.
After creating the table, run the word count map reduce job.
- [user1@instance accumulo]$ bin/tool.sh lib/accumulo-examples-*.jar org.apache.accumulo.examples.mapreduce.WordCount instance zookeepers /user/user1/wc wordCount -u username -p password
+ [user1@instance accumulo]$ bin/tool.sh lib/accumulo-examples-*[^c].jar org.apache.accumulo.examples.mapreduce.WordCount instance zookeepers /user/user1/wc wordCount -u username -p password
11/02/07 18:20:11 INFO input.FileInputFormat: Total input paths to process : 1
11/02/07 18:20:12 INFO mapred.JobClient: Running job: job_201102071740_0003
Modified: incubator/accumulo/branches/1.3/src/assemble/build.sh
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.3/src/assemble/build.sh?rev=1196106&r1=1196105&r2=1196106&view=diff
==============================================================================
--- incubator/accumulo/branches/1.3/src/assemble/build.sh (original)
+++ incubator/accumulo/branches/1.3/src/assemble/build.sh Tue Nov 1 16:37:23 2011
@@ -41,9 +41,10 @@ runAt() {
run mvn -U -P distclean clean
mvn rat:check
COUNT=`grep '!????' target/rat.txt | wc -l`
-if [ "$COUNT" -ne 31 ]
+EXPECTED=19
+if [ "$COUNT" -ne $EXPECTED ]
then
- fail expected 32 files missing licenses, but saw "$COUNT"
+ fail expected $EXPECTED files missing licenses, but saw "$COUNT"
fi
run mvn package javadoc:aggregate javadoc:jar source:jar
runAt ./src/server/src/main/c++ make