You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bu...@apache.org on 2014/03/17 22:10:31 UTC
svn commit: r902049 [1/3] - in /websites/staging/accumulo/trunk/content: ./
1.4/ 1.4/examples/ 1.4/user_manual/ 1.5/ 1.5/examples/
Author: buildbot
Date: Mon Mar 17 21:10:31 2014
New Revision: 902049
Log:
Staging update by buildbot for accumulo
Added:
websites/staging/accumulo/trunk/content/1.4/user_manual/Development_Clients.html
Modified:
websites/staging/accumulo/trunk/content/ (props changed)
websites/staging/accumulo/trunk/content/1.4/accumulo_user_manual.pdf
websites/staging/accumulo/trunk/content/1.4/examples/bulkIngest.html
websites/staging/accumulo/trunk/content/1.4/examples/dirlist.html
websites/staging/accumulo/trunk/content/1.4/examples/filedata.html
websites/staging/accumulo/trunk/content/1.4/examples/mapred.html
websites/staging/accumulo/trunk/content/1.4/user_manual/Administration.html
websites/staging/accumulo/trunk/content/1.4/user_manual/Contents.html
websites/staging/accumulo/trunk/content/1.4/user_manual/Table_Configuration.html
websites/staging/accumulo/trunk/content/1.4/user_manual/Writing_Accumulo_Clients.html
websites/staging/accumulo/trunk/content/1.4/user_manual/accumulo_user_manual.html
websites/staging/accumulo/trunk/content/1.4/user_manual/index.html
websites/staging/accumulo/trunk/content/1.5/accumulo_user_manual.html
websites/staging/accumulo/trunk/content/1.5/examples/filedata.html
Propchange: websites/staging/accumulo/trunk/content/
------------------------------------------------------------------------------
--- cms:source-revision (original)
+++ cms:source-revision Mon Mar 17 21:10:31 2014
@@ -1 +1 @@
-1577778
+1578577
Modified: websites/staging/accumulo/trunk/content/1.4/accumulo_user_manual.pdf
==============================================================================
Binary files - no diff available.
Modified: websites/staging/accumulo/trunk/content/1.4/examples/bulkIngest.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/examples/bulkIngest.html (original)
+++ websites/staging/accumulo/trunk/content/1.4/examples/bulkIngest.html Mon Mar 17 21:10:31 2014
@@ -101,7 +101,7 @@ accumulo instance name, and a comma-sepa
<div class="codehilite"><pre>$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">mapreduce</span><span class="p">.</span><span class="n">bulk</span><span class="p">.</span><span class="n">SetupTable</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">test_bulk</span> <span class="n">row_00000333</span> <span class="n">row_00000666</span>
$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">mapreduce</span><span class="p">.</span><span class="n">bulk</span><span class="p">.</span><span class="n">GenerateTestData</span> 0 1000 <span class="n">bulk</span><span class="o">/</span><span class="n">test_1</span><span class="p">.</span><span class="n">txt</span>
-$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">tool</span><span class="p">.</span><span class="n">sh</span> <span class="n">lib</span><span class="o">/</span><span class="n">examples</span><span class="o">-</span><span class="n">simple</span><span class="o">-*</span><span class="p">[</span>^<span class="n">c</span><span class="p">].</span><span class="n">jar</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">mapreduce</span><span class="p">.</span><span class="n">bulk</span><span class="p">.</span><span class="n">BulkIngestExample</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">test_bulk</span> <span
class="n">bulk</span> <span class="n">tmp</span><span class="o">/</span><span class="n">bulkWork</span>
+$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">tool</span><span class="p">.</span><span class="n">sh</span> <span class="n">lib</span><span class="o">/</span><span class="n">examples</span><span class="o">-</span><span class="n">simple</span><span class="o">-*</span><span class="p">[</span>^<span class="n">cs</span><span class="p">].</span><span class="n">jar</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">mapreduce</span><span class="p">.</span><span class="n">bulk</span><span class="p">.</span><span class="n">BulkIngestExample</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">test_bulk</span> <span
class="n">bulk</span> <span class="n">tmp</span><span class="o">/</span><span class="n">bulkWork</span>
$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">mapreduce</span><span class="p">.</span><span class="n">bulk</span><span class="p">.</span><span class="n">VerifyIngest</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">test_bulk</span> 0 1000
</pre></div>
Modified: websites/staging/accumulo/trunk/content/1.4/examples/dirlist.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/examples/dirlist.html (original)
+++ websites/staging/accumulo/trunk/content/1.4/examples/dirlist.html Mon Mar 17 21:10:31 2014
@@ -112,14 +112,19 @@
<p>This may take some time if there are large files in the /local/username/workspace directory. If you use 0 instead of 100000 on the command line, the ingest will run much faster, but it will not put any file data into Accumulo (the dataTable will be empty).
Note that running this example will create tables dirTable, indexTable, and dataTable in Accumulo that you should delete when you have completed the example.
If you modify a file or add new files in the directory ingested (e.g. /local/username/workspace), you can run Ingest again to add new information into the Accumulo tables.</p>
-<p>To browse the data ingested, use Viewer.java. Be sure to give the "username" user the authorizations to see the data (in this case, run "setauths -u username -s exampleVis" in the shell, and use the string "exampleVis" as the "auths" in command lines below).</p>
-<div class="codehilite"><pre>$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">dirlist</span><span class="p">.</span><span class="n">Viewer</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">dirTable</span> <span class="n">dataTable</span> <span class="n">auths</span> <span class="o">/</span><span class="n">local</span><span class="o">/</span><span class="n">username</span><span class="o">/</span><span class="n">workspace</span>
+<p>To browse the data ingested, use Viewer.java. Be sure to give the "username" user the authorizations to see the data (in this case, run</p>
+<div class="codehilite"><pre>$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">shell</span> <span class="o">-</span><span class="n">u</span> <span class="n">root</span> <span class="o">-</span><span class="n">e</span> <span class="s">'setauths -u username -s exampleVis'</span>
+</pre></div>
+
+
+<p>then run the Viewer:</p>
+<div class="codehilite"><pre>$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">dirlist</span><span class="p">.</span><span class="n">Viewer</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">dirTable</span> <span class="n">dataTable</span> <span class="n">exampleVis</span> <span class="o">/</span><span class="n">local</span><span class="o">/</span><span class="n">username</span><span class="o">/</span><span class="n">workspace</span>
</pre></div>
<p>To list the contents of specific directories, use QueryUtil.java.</p>
-<div class="codehilite"><pre>$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">dirlist</span><span class="p">.</span><span class="n">QueryUtil</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">dirTable</span> <span class="n">auths</span> <span class="o">/</span><span class="n">local</span><span class="o">/</span><span class="n">username</span>
-$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">dirlist</span><span class="p">.</span><span class="n">QueryUtil</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">dirTable</span> <span class="n">auths</span> <span class="o">/</span><span class="n">local</span><span class="o">/</span><span class="n">username</span><span class="o">/</span><span class="n">workspace</span>
+<div class="codehilite"><pre>$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">dirlist</span><span class="p">.</span><span class="n">QueryUtil</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">dirTable</span> <span class="n">exampleVis</span> <span class="o">/</span><span class="n">local</span><span class="o">/</span><span class="n">username</span>
+$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">dirlist</span><span class="p">.</span><span class="n">QueryUtil</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">dirTable</span> <span class="n">exampleVis</span> <span class="o">/</span><span class="n">local</span><span class="o">/</span><span class="n">username</span><span class="o">/</span><span class="n">workspace</span>
</pre></div>
Modified: websites/staging/accumulo/trunk/content/1.4/examples/filedata.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/examples/filedata.html (original)
+++ websites/staging/accumulo/trunk/content/1.4/examples/filedata.html Mon Mar 17 21:10:31 2014
@@ -98,8 +98,8 @@ The example has the following classes:</
<li>ChunkCombiner - An Iterator that dedupes file data and sets their visibilities to a combined visibility based on current references to the file data.</li>
<li>ChunkInputFormat - An Accumulo InputFormat that provides keys containing file info (List<Entry<Key,Value>>) and values with an InputStream over the file (ChunkInputStream).</li>
<li>ChunkInputStream - An input stream over file data stored in Accumulo.</li>
-<li>FileDataIngest - Takes a list of files and archives them into Accumulo keyed on the SHA1 hashes of the files.</li>
-<li>FileDataQuery - Retrieves file data based on the SHA1 hash of the file. (Used by the dirlist.Viewer.)</li>
+<li>FileDataIngest - Takes a list of files and archives them into Accumulo keyed on hashes of the files.</li>
+<li>FileDataQuery - Retrieves file data based on the hash of the file. (Used by the dirlist.Viewer.)</li>
<li>KeyUtil - A utility for creating and parsing null-byte separated strings into/from Text objects.</li>
<li>VisibilityCombiner - A utility for merging visibilities into the form (VIS1)|(VIS2)|...</li>
</ul>
@@ -115,7 +115,7 @@ The example has the following classes:</
<p>Run the CharacterHistogram MapReduce to add some information about the file.</p>
-<div class="codehilite"><pre>$ <span class="n">bin</span><span class="o">/</span><span class="n">tool</span><span class="p">.</span><span class="n">sh</span> <span class="n">lib</span><span class="o">/</span><span class="n">examples</span><span class="o">-</span><span class="n">simple</span><span class="o">*</span><span class="p">[</span>^<span class="n">c</span><span class="p">].</span><span class="n">jar</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">filedata</span><span class="p">.</span><span class="n">CharacterHistogram</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">dataTable</span> <span class="n">exampleVis</span> <span class="n">exam
pleVis</span>
+<div class="codehilite"><pre>$ <span class="n">bin</span><span class="o">/</span><span class="n">tool</span><span class="p">.</span><span class="n">sh</span> <span class="n">lib</span><span class="o">/</span><span class="n">examples</span><span class="o">-</span><span class="n">simple</span><span class="o">*</span><span class="p">[</span>^<span class="n">cs</span><span class="p">].</span><span class="n">jar</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">filedata</span><span class="p">.</span><span class="n">CharacterHistogram</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="n">username</span> <span class="n">password</span> <span class="n">dataTable</span> <span class="n">exampleVis</span> <span class="n">exa
mpleVis</span>
</pre></div>
Modified: websites/staging/accumulo/trunk/content/1.4/examples/mapred.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/examples/mapred.html (original)
+++ websites/staging/accumulo/trunk/content/1.4/examples/mapred.html Mon Mar 17 21:10:31 2014
@@ -125,7 +125,7 @@ for the column family count.</p>
<p>After creating the table, run the word count map reduce job.</p>
-<div class="codehilite"><pre>$ <span class="n">bin</span><span class="o">/</span><span class="n">tool</span><span class="p">.</span><span class="n">sh</span> <span class="n">lib</span><span class="o">/</span><span class="n">examples</span><span class="o">-</span><span class="n">simple</span><span class="o">*</span><span class="p">[</span>^<span class="n">c</span><span class="p">].</span><span class="n">jar</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">mapreduce</span><span class="p">.</span><span class="n">WordCount</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">username</span><span class="o">/</span><span class="n">wc</span> <sp
an class="n">wordCount</span> <span class="o">-</span><span class="n">u</span> <span class="n">username</span> <span class="o">-</span><span class="n">p</span> <span class="n">password</span>
+<div class="codehilite"><pre>$ <span class="n">bin</span><span class="o">/</span><span class="n">tool</span><span class="p">.</span><span class="n">sh</span> <span class="n">lib</span><span class="o">/</span><span class="n">examples</span><span class="o">-</span><span class="n">simple</span><span class="o">*</span><span class="p">[</span>^<span class="n">cs</span><span class="p">].</span><span class="n">jar</span> <span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">examples</span><span class="p">.</span><span class="n">simple</span><span class="p">.</span><span class="n">mapreduce</span><span class="p">.</span><span class="n">WordCount</span> <span class="n">instance</span> <span class="n">zookeepers</span> <span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">username</span><span class="o">/</span><span class="n">wc</span> <s
pan class="n">wordCount</span> <span class="o">-</span><span class="n">u</span> <span class="n">username</span> <span class="o">-</span><span class="n">p</span> <span class="n">password</span>
11<span class="o">/</span>02<span class="o">/</span>07 18<span class="p">:</span>20<span class="p">:</span>11 <span class="n">INFO</span> <span class="n">input</span><span class="p">.</span><span class="n">FileInputFormat</span><span class="p">:</span> <span class="n">Total</span> <span class="n">input</span> <span class="n">paths</span> <span class="n">to</span> <span class="n">process</span> <span class="p">:</span> 1
11<span class="o">/</span>02<span class="o">/</span>07 18<span class="p">:</span>20<span class="p">:</span>12 <span class="n">INFO</span> <span class="n">mapred</span><span class="p">.</span><span class="n">JobClient</span><span class="p">:</span> <span class="n">Running</span> <span class="n">job</span><span class="p">:</span> <span class="n">job_201102071740_0003</span>
Modified: websites/staging/accumulo/trunk/content/1.4/user_manual/Administration.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/user_manual/Administration.html (original)
+++ websites/staging/accumulo/trunk/content/1.4/user_manual/Administration.html Mon Mar 17 21:10:31 2014
@@ -106,25 +106,25 @@
<li><a href="Administration.html#Recovery">Recovery</a></li>
</ul>
<hr />
-<h2 id="wzxhzdk4wzxhzdk5-administration"><a id=Administration></a> Administration</h2>
-<h2 id="wzxhzdk6wzxhzdk7-hardware"><a id=Hardware></a> Hardware</h2>
+<h2 id="wzxhzdk6wzxhzdk7-administration"><a id=Administration></a> Administration</h2>
+<h2 id="wzxhzdk8wzxhzdk9-hardware"><a id=Hardware></a> Hardware</h2>
<p>Because we are running essentially two or three systems simultaneously layered across the cluster: HDFS, Accumulo and MapReduce, it is typical for hardware to consist of 4 to 8 cores, and 8 to 32 GB RAM. This is so each running process can have at least one core and 2 - 4 GB each. </p>
<p>One core running HDFS can typically keep 2 to 4 disks busy, so each machine may typically have as little as 2 x 300GB disks and as much as 4 x 1TB or 2TB disks. </p>
<p>It is possible to do with less than this, such as with 1u servers with 2 cores and 4GB each, but in this case it is recommended to only run up to two processes per machine - i.e. DataNode and TabletServer or DataNode and MapReduce worker but not all three. The constraint here is having enough available heap space for all the processes on a machine. </p>
-<h2 id="wzxhzdk8wzxhzdk9-network"><a id=Network></a> Network</h2>
+<h2 id="wzxhzdk10wzxhzdk11-network"><a id=Network></a> Network</h2>
<p>Accumulo communicates via remote procedure calls over TCP/IP for both passing data and control messages. In addition, Accumulo uses HDFS clients to communicate with HDFS. To achieve good ingest and query performance, sufficient network bandwidth must be available between any two machines. </p>
-<h2 id="wzxhzdk10wzxhzdk11-installation"><a id=Installation></a> Installation</h2>
+<h2 id="wzxhzdk12wzxhzdk13-installation"><a id=Installation></a> Installation</h2>
<p>Choose a directory for the Accumulo installation. This directory will be referenced by the environment variable $ACCUMULO_HOME. Run the following: </p>
<div class="codehilite"><pre>$ <span class="n">tar</span> <span class="n">xzf</span> $<span class="n">ACCUMULO_HOME</span><span class="o">/</span><span class="n">accumulo</span><span class="p">.</span><span class="n">tar</span><span class="p">.</span><span class="n">gz</span>
</pre></div>
<p>Repeat this step at each machine within the cluster. Usually all machines have the same $ACCUMULO_HOME. </p>
-<h2 id="wzxhzdk12wzxhzdk13-dependencies"><a id=Dependencies></a> Dependencies</h2>
+<h2 id="wzxhzdk14wzxhzdk15-dependencies"><a id=Dependencies></a> Dependencies</h2>
<p>Accumulo requires HDFS and ZooKeeper to be configured and running before starting. Password-less SSH should be configured between at least the Accumulo master and TabletServer machines. It is also a good idea to run Network Time Protocol (NTP) within the cluster to ensure nodes' clocks don't get too out of sync, which can cause problems with automatically timestamped data. Accumulo will remove from the set of TabletServers those machines whose times differ too much from the master's. </p>
-<h2 id="wzxhzdk14wzxhzdk15-configuration"><a id=Configuration></a> Configuration</h2>
+<h2 id="wzxhzdk16wzxhzdk17-configuration"><a id=Configuration></a> Configuration</h2>
<p>Accumulo is configured by editing several Shell and XML files found in $ACCUMULO_HOME/conf. The structure closely resembles Hadoop's configuration files. </p>
-<h3 id="wzxhzdk16wzxhzdk17-edit-confaccumulo-envsh"><a id=Edit_conf/accumulo-env.sh></a> Edit conf/accumulo-env.sh</h3>
+<h3 id="wzxhzdk18wzxhzdk19-edit-confaccumulo-envsh"><a id=Edit_conf/accumulo-env.sh></a> Edit conf/accumulo-env.sh</h3>
<p>Accumulo needs to know where to find the software it depends on. Edit accumulo-env.sh and specify the following: </p>
<ol>
<li>Enter the location of the installation directory of Accumulo for $ACCUMULO_HOME</li>
@@ -136,7 +136,7 @@
<p>By default Accumulo TabletServers are set to use 1GB of memory. You may change this by altering the value of $ACCUMULO_TSERVER_OPTS. Note the syntax is that of the Java JVM command line options. This value should be less than the physical memory of the machines running TabletServers. </p>
<p>There are similar options for the master's memory usage and the garbage collector process. Reduce these if they exceed the physical RAM of your hardware and increase them, within the bounds of the physical RAM, if a process fails because of insufficient memory. </p>
<p>Note that you will be specifying the Java heap space in accumulo-env.sh. You should make sure that the total heap space used for the Accumulo tserver and the Hadoop DataNode and TaskTracker is less than the available memory on each slave node in the cluster. On large clusters, it is recommended that the Accumulo master, Hadoop NameNode, secondary NameNode, and Hadoop JobTracker all be run on separate machines to allow them to use more heap space. If you are running these on the same machine on a small cluster, likewise make sure their heap space settings fit within the available memory. </p>
-<h3 id="wzxhzdk18wzxhzdk19-cluster-specification"><a id=Cluster_Specification></a> Cluster Specification</h3>
+<h3 id="wzxhzdk20wzxhzdk21-cluster-specification"><a id=Cluster_Specification></a> Cluster Specification</h3>
<p>On the machine that will serve as the Accumulo master: </p>
<ol>
<li>Write the IP address or domain name of the Accumulo Master to the <br />
@@ -145,7 +145,7 @@ $ACCUMULO_HOME/conf/masters file. </li>
$ACCUMULO_HOME/conf/slaves, one per line. </li>
</ol>
<p>Note that if using domain names rather than IP addresses, DNS must be configured properly for all machines participating in the cluster. DNS can be a confusing source of errors. </p>
-<h3 id="wzxhzdk20wzxhzdk21-accumulo-settings"><a id=Accumulo_Settings></a> Accumulo Settings</h3>
+<h3 id="wzxhzdk22wzxhzdk23-accumulo-settings"><a id=Accumulo_Settings></a> Accumulo Settings</h3>
<p>Specify appropriate values for the following settings in <br />
$ACCUMULO_HOME/conf/accumulo-site.xml : </p>
<div class="codehilite"><pre><span class="nt"><property></span>
@@ -164,27 +164,43 @@ $ACCUMULO_HOME/conf/accumulo-site.xml :
<p>This enables Accumulo to find ZooKeeper. Accumulo uses ZooKeeper to coordinate settings between processes and helps finalize TabletServer failure. </p>
<p>Accumulo records all changes to tables to a write-ahead log before committing them to the table. The `walog' setting specifies the local directory on each machine to which write-ahead logs are written. This directory should exist on all machines acting as TabletServers. </p>
<p>Some settings can be modified via the Accumulo shell and take effect immediately. However, any settings that should be persisted across system restarts must be recorded in the accumulo-site.xml file. </p>
-<h3 id="wzxhzdk22wzxhzdk23-deploy-configuration"><a id=Deploy_Configuration></a> Deploy Configuration</h3>
+<h3 id="wzxhzdk24wzxhzdk25-deploy-configuration"><a id=Deploy_Configuration></a> Deploy Configuration</h3>
<p>Copy the masters, slaves, accumulo-env.sh, and if necessary, accumulo-site.xml from the <br />
$ACCUMULO_HOME/conf/ directory on the master to all the machines specified in the slaves file. </p>
-<h2 id="wzxhzdk24wzxhzdk25-initialization"><a id=Initialization></a> Initialization</h2>
+<h2 id="wzxhzdk26wzxhzdk27-initialization"><a id=Initialization></a> Initialization</h2>
<p>Accumulo must be initialized to create the structures it uses internally to locate data across the cluster. HDFS is required to be configured and running before Accumulo can be initialized. </p>
<p>Once HDFS is started, initialization can be performed by executing <br />
$ACCUMULO_HOME/bin/accumulo init . This script will prompt for a name for this instance of Accumulo. The instance name is used to identify a set of tables and instance-specific settings. The script will then write some information into HDFS so Accumulo can start properly. </p>
<p>The initialization script will prompt you to set a root password. Once Accumulo is initialized it can be started. </p>
-<h2 id="wzxhzdk26wzxhzdk27-running"><a id=Running></a> Running</h2>
-<h3 id="wzxhzdk28wzxhzdk29-starting-accumulo"><a id=Starting_Accumulo></a> Starting Accumulo</h3>
+<h2 id="wzxhzdk28wzxhzdk29-running"><a id=Running></a> Running</h2>
+<h3 id="wzxhzdk30wzxhzdk31-starting-accumulo"><a id=Starting_Accumulo></a> Starting Accumulo</h3>
<p>Make sure Hadoop is configured on all of the machines in the cluster, including access to a shared HDFS instance. Make sure HDFS and ZooKeeper are running. Make sure ZooKeeper is configured and running on at least one machine in the cluster. Start Accumulo using the bin/start-all.sh script. </p>
<p>To verify that Accumulo is running, check the Status page as described under <em>Monitoring</em>. In addition, the Shell can provide some information about the status of tables via reading the !METADATA table. </p>
-<h3 id="wzxhzdk30wzxhzdk31-stopping-accumulo"><a id=Stopping_Accumulo></a> Stopping Accumulo</h3>
+<h3 id="wzxhzdk32wzxhzdk33-stopping-accumulo"><a id=Stopping_Accumulo></a> Stopping Accumulo</h3>
<p>To shutdown cleanly, run bin/stop-all.sh and the master will orchestrate the shutdown of all the tablet servers. Shutdown waits for all minor compactions to finish, so it may take some time for particular configurations. </p>
-<h2 id="wzxhzdk32wzxhzdk33-monitoring"><a id=Monitoring></a> Monitoring</h2>
+<h3 id="wzxhzdk34wzxhzdk35-adding-a-node"><a id=Adding_a_Node></a> Adding a Node</h3>
+<p>Update your $ACCUMULO_HOME/conf/slaves (or $ACCUMULO_CONF_DIR/slaves) file to account for the addition. </p>
+<div class="codehilite"><pre>$<span class="n">ACCUMULO_HOME</span><span class="o">/</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">admin</span> <span class="n">start</span> <span class="o"><</span><span class="n">host</span><span class="p">(</span><span class="n">s</span><span class="p">)</span><span class="o">></span> <span class="p">{</span><span class="o"><</span><span class="n">host</span><span class="o">></span> <span class="p">...}</span>
+</pre></div>
+
+
+<p>Alternatively, you can ssh to each of the hosts you want to add and run $ACCUMULO_HOME/bin/start-here.sh. </p>
+<p>Make sure the host in question has the new configuration, or else the tablet server won't start; at a minimum this needs to be on the host(s) being added, but in practice it's good to ensure consistent configuration across all nodes. </p>
+<h3 id="wzxhzdk36wzxhzdk37-decomissioning-a-node"><a id=Decomissioning_a_Node></a> Decomissioning a Node</h3>
+<p>If you need to take a node out of operation, you can trigger a graceful shutdown of a tablet server. Accumulo will automatically rebalance the tablets across the available tablet servers. </p>
+<div class="codehilite"><pre>$<span class="n">ACCUMULO_HOME</span><span class="o">/</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">admin</span> <span class="n">stop</span> <span class="o"><</span><span class="n">host</span><span class="p">(</span><span class="n">s</span><span class="p">)</span><span class="o">></span> <span class="p">{</span><span class="o"><</span><span class="n">host</span><span class="o">></span> <span class="p">...}</span>
+</pre></div>
+
+
+<p>Alternatively, you can ssh to each of the hosts you want to remove and run $ACCUMULO_HOME/bin/stop-here.sh. </p>
+<p>Be sure to update your $ACCUMULO_HOME/conf/slaves (or $ACCUMULO_CONF_DIR/slaves) file to account for the removal of these hosts. Bear in mind that the monitor will not re-read the slaves file automatically, so it will report the decomissioned servers as down; it's recommended that you restart the monitor so that the node list is up to date. </p>
+<h2 id="wzxhzdk38wzxhzdk39-monitoring"><a id=Monitoring></a> Monitoring</h2>
<p>The Accumulo Master provides an interface for monitoring the status and health of Accumulo components. This interface can be accessed by pointing a web browser to <br />
http://accumulomaster:50095/status</p>
-<h2 id="wzxhzdk34wzxhzdk35-logging"><a id=Logging></a> Logging</h2>
+<h2 id="wzxhzdk40wzxhzdk41-logging"><a id=Logging></a> Logging</h2>
<p>Accumulo processes each write to a set of log files. By default these are found under <br />
$ACCUMULO/logs/. </p>
-<h2 id="wzxhzdk36wzxhzdk37-recovery"><a id=Recovery></a> Recovery</h2>
+<h2 id="wzxhzdk42wzxhzdk43-recovery"><a id=Recovery></a> Recovery</h2>
<p>In the event of TabletServer failure or error on shutting Accumulo down, some mutations may not have been minor compacted to HDFS properly. In this case, Accumulo will automatically reapply such mutations from the write-ahead log either when the tablets from the failed server are reassigned by the Master, in the case of a single TabletServer failure or the next time Accumulo starts, in the event of failure during shutdown. </p>
<p>Recovery is performed by asking the loggers to copy their write-ahead logs into HDFS. As the logs are copied, they are also sorted, so that tablets can easily find their missing updates. The copy/sort status of each file is displayed on Accumulo monitor status page. Once the recovery is complete any tablets involved should return to an ``online" state. Until then those tablets will be unavailable to clients. </p>
<p>The Accumulo client library is configured to retry failed mutations and in many cases clients will be able to continue processing after the recovery process without throwing an exception. </p>
Modified: websites/staging/accumulo/trunk/content/1.4/user_manual/Contents.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/user_manual/Contents.html (original)
+++ websites/staging/accumulo/trunk/content/1.4/user_manual/Contents.html Mon Mar 17 21:10:31 2014
@@ -145,6 +145,23 @@
<li><a href="Writing_Accumulo_Clients.html#BatchScanner">BatchScanner</a></li>
</ul>
</li>
+<li>
+<p><a href="Writing_Accumulo_Clients.html#Proxy">Proxy</a></p>
+<ul>
+<li><a href="Writing_Accumulo_Clients.html#Prequisites">Prequisites</a></li>
+<li><a href="Writing_Accumulo_Clients.html#Configuration">Configuration</a></li>
+<li><a href="Writing_Accumulo_Clients.html#Running_the_Proxy_Server">Running the Proxy Server</a></li>
+<li><a href="Writing_Accumulo_Clients.html#Creating_a_Proxy_Client">Creating a Proxy Client</a></li>
+<li><a href="Writing_Accumulo_Clients.html#Using_a_Proxy_Client">Using a Proxy Client</a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li>
+<p><a href="Development_Clients.html">Development Clients</a></p>
+<ul>
+<li><a href="Development_Clients.html#Mock_Accumulo">Mock Accumulo</a></li>
+<li><a href="Development_Clients.html#Mini_Accumulo_Cluster">Mini Accumulo Cluster</a></li>
</ul>
</li>
<li>
@@ -258,6 +275,8 @@
<ul>
<li><a href="Administration.html#Starting_Accumulo">Starting Accumulo</a></li>
<li><a href="Administration.html#Stopping_Accumulo">Stopping Accumulo</a></li>
+<li><a href="Administration.html#Adding_a_Node">Adding a Node</a></li>
+<li><a href="Administration.html#Decomissioning_a_Node">Decomissioning a Node</a></li>
</ul>
</li>
<li>
Added: websites/staging/accumulo/trunk/content/1.4/user_manual/Development_Clients.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/user_manual/Development_Clients.html (added)
+++ websites/staging/accumulo/trunk/content/1.4/user_manual/Development_Clients.html Mon Mar 17 21:10:31 2014
@@ -0,0 +1,190 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE- 2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <link href="/css/accumulo.css" rel="stylesheet" type="text/css">
+ <title>Apache Accumulo User Manual: Development Clients</title>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+
+ <script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-21103458-6']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_setAllowLinker', true]);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+ </script>
+
+
+</head>
+
+<body>
+ <div id="banner">
+ </div>
+
+ <div id="navigation">
+ <h1 id="project">Project</h1>
+<ul>
+<li><a href="/">Home</a></li>
+<li><a href="/downloads">Downloads</a></li>
+<li><a href="/notable_features.html">Features</a></li>
+<li><a href="http://www.apache.org/licenses/LICENSE-2.0">License</a></li>
+</ul>
+<h1 id="community">Community</h1>
+<ul>
+<li><a href="/get_involved.html">Get Involved</a></li>
+<li><a href="/mailing_list.html">Mailing Lists</a></li>
+<li><a href="/people.html">People</a></li>
+</ul>
+<h1 id="development">Development</h1>
+<ul>
+<li><a href="/source.html">Source & Guide</a></li>
+<li><a href="/git.html">Git WIP</a></li>
+<li><a href="/contrib.html">Contrib Projects</a></li>
+<li><a href="/releasing.html">Making Releases</a></li>
+<li><a href="https://issues.apache.org/jira/browse/accumulo">Issues</a></li>
+<li><a href="https://builds.apache.org/view/A-D/view/Accumulo/">Builds</a></li>
+</ul>
+<h1 id="documentation">Documentation</h1>
+<ul>
+<li>Manual <a href="/1.4/user_manual">1.4</a> / <a href="/1.5/accumulo_user_manual.html">1.5</a></li>
+<li>Javadoc <a href="/1.4/apidocs">1.4</a> / <a href="/1.5/apidocs">1.5</a></li>
+<li>Examples <a href="/1.4/examples">1.4</a> / <a href="/1.5/examples">1.5</a></li>
+<li><a href="/screenshots.html">Screenshots</a></li>
+<li><a href="/papers.html">Papers & Other Links</a></li>
+<li><a href="/glossary.html">Glossary</a></li>
+</ul>
+<h1 id="asf-links">ASF links</h1>
+<ul>
+<li><a href="http://www.apache.org">Apache Software Foundation</a></li>
+<li><a href="http://www.apache.org/foundation/sponsorship.html">Sponsorship</a></li>
+<li><a href="http://www.apache.org/security/">Security</a></li>
+<li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+</ul>
+ </div>
+
+ <div id="bannertext">
+ <img id="logo" alt="Apache Accumulo" src="/images/accumulo-logo.png"/>™
+ </div>
+
+ <div id="content">
+ <h1 class="title">Apache Accumulo User Manual: Development Clients</h1>
+ <p><strong> Next:</strong> <a href="Table_Configuration.html">Table Configuration</a> <strong> Up:</strong> <a href="accumulo_user_manual.html">Apache Accumulo User Manual Version 1.4</a> <strong> Previous:</strong> <a href="Writing_Accumulo_Clients.html">Writing Accumulo Clients</a> <strong> <a href="Contents.html">Contents</a></strong> </p>
+<p><a id=CHILD_LINKS></a><strong>Subsections</strong></p>
+<ul>
+<li><a href="Development_Clients.html#Mock_Accumulo">Mock Accumulo</a></li>
+<li><a href="Development_Clients.html#Mini_Accumulo_Cluster">Mini Accumulo Cluster</a></li>
+</ul>
+<hr />
+<h2 id="wzxhzdk9wzxhzdk10-development-clients"><a id=Development_Clients></a> Development Clients</h2>
+<p>Normally, Accumulo consists of lots of moving parts. Even a stand-alone version of Accumulo requires Hadoop, Zookeeper, the Accumulo master, a tablet server, etc. If you want to write a unit test that uses Accumulo, you need a lot of infrastructure in place before your test can run. </p>
+<h2 id="wzxhzdk11wzxhzdk12-mock-accumulo"><a id=Mock_Accumulo></a> Mock Accumulo</h2>
+<p>Mock Accumulo supplies mock implementations for much of the client API. It presently does not enforce users, logins, permissions, etc. It does support Iterators and Combiners. Note that MockAccumulo holds all data in memory, and will not retain any data or settings between runs. </p>
+<p>While normal interaction with the Accumulo client looks like this: </p>
+<div class="codehilite"><pre><span class="n">Instance</span> <span class="n">instance</span> <span class="p">=</span> <span class="n">new</span> <span class="n">ZooKeeperInstance</span><span class="p">(...);</span>
+<span class="n">Connector</span> <span class="n">conn</span> <span class="p">=</span> <span class="n">instance</span><span class="p">.</span><span class="n">getConnector</span><span class="p">(</span><span class="n">user</span><span class="p">,</span> <span class="n">passwd</span><span class="p">);</span>
+</pre></div>
+
+
+<p>To interact with the MockAccumulo, just replace the ZooKeeperInstance with MockInstance: </p>
+<div class="codehilite"><pre><span class="n">Instance</span> <span class="n">instance</span> <span class="p">=</span> <span class="n">new</span> <span class="n">MockInstance</span><span class="p">();</span>
+</pre></div>
+
+
+<p>In fact, you can use the "-fake" option to the Accumulo shell and interact with MockAccumulo: </p>
+<div class="codehilite"><pre>$ <span class="o">./</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">shell</span> <span class="o">--</span><span class="n">fake</span> <span class="o">-</span><span class="n">u</span> <span class="n">root</span> <span class="o">-</span><span class="n">p</span> <span class="s">''</span>
+
+<span class="n">Shell</span> <span class="o">-</span> <span class="n">Apache</span> <span class="n">Accumulo</span> <span class="n">Interactive</span> <span class="n">Shell</span>
+<span class="o">-</span>
+<span class="o">-</span> <span class="n">version</span><span class="p">:</span> 1<span class="p">.</span>4<span class="p">.</span>4
+<span class="o">-</span> <span class="n">instance</span> <span class="n">name</span><span class="p">:</span> <span class="n">mock</span><span class="o">-</span><span class="n">instance</span>
+<span class="o">-</span> <span class="n">instance</span> <span class="n">id</span><span class="p">:</span> <span class="n">mock</span><span class="o">-</span><span class="n">instance</span><span class="o">-</span><span class="n">id</span>
+<span class="o">-</span>
+<span class="o">-</span> <span class="n">type</span> <span class="s">'help'</span> <span class="k">for</span> <span class="n">a</span> <span class="n">list</span> <span class="n">of</span> <span class="n">available</span> <span class="n">commands</span>
+<span class="o">-</span>
+<span class="n">root</span><span class="p">@</span><span class="n">mock</span><span class="o">-</span><span class="n">instance</span><span class="o">></span> <span class="n">createtable</span> <span class="n">test</span>
+<span class="n">root</span><span class="p">@</span><span class="n">mock</span><span class="o">-</span><span class="n">instance</span> <span class="n">test</span><span class="o">></span> <span class="n">insert</span> <span class="n">row1</span> <span class="n">cf</span> <span class="n">cq</span> <span class="n">value</span>
+<span class="n">root</span><span class="p">@</span><span class="n">mock</span><span class="o">-</span><span class="n">instance</span> <span class="n">test</span><span class="o">></span> <span class="n">insert</span> <span class="n">row2</span> <span class="n">cf</span> <span class="n">cq</span> <span class="n">value2</span>
+<span class="n">root</span><span class="p">@</span><span class="n">mock</span><span class="o">-</span><span class="n">instance</span> <span class="n">test</span><span class="o">></span> <span class="n">insert</span> <span class="n">row3</span> <span class="n">cf</span> <span class="n">cq</span> <span class="n">value3</span>
+<span class="n">root</span><span class="p">@</span><span class="n">mock</span><span class="o">-</span><span class="n">instance</span> <span class="n">test</span><span class="o">></span> <span class="n">scan</span>
+<span class="n">row1</span> <span class="n">cf</span><span class="p">:</span><span class="n">cq</span> <span class="p">[]</span> <span class="n">value</span>
+<span class="n">row2</span> <span class="n">cf</span><span class="p">:</span><span class="n">cq</span> <span class="p">[]</span> <span class="n">value2</span>
+<span class="n">row3</span> <span class="n">cf</span><span class="p">:</span><span class="n">cq</span> <span class="p">[]</span> <span class="n">value3</span>
+<span class="n">root</span><span class="p">@</span><span class="n">mock</span><span class="o">-</span><span class="n">instance</span> <span class="n">test</span><span class="o">></span> <span class="n">scan</span> <span class="o">-</span><span class="n">b</span> <span class="n">row2</span> <span class="o">-</span><span class="n">e</span> <span class="n">row2</span>
+<span class="n">row2</span> <span class="n">cf</span><span class="p">:</span><span class="n">cq</span> <span class="p">[]</span> <span class="n">value2</span>
+<span class="n">root</span><span class="p">@</span><span class="n">mock</span><span class="o">-</span><span class="n">instance</span> <span class="n">test</span><span class="o">></span>
+</pre></div>
+
+
+<p>When testing Map Reduce jobs, you can also set the Mock Accumulo on the AccumuloInputFormat and AccumuloOutputFormat classes: </p>
+<div class="codehilite"><pre><span class="c1">// ... set up job configuration</span>
+<span class="n">AccumuloInputFormat</span><span class="p">.</span><span class="n">setMockInstance</span><span class="p">(</span><span class="n">job</span><span class="p">,</span> <span class="s">"mockInstance"</span><span class="p">);</span>
+<span class="n">AccumuloOutputFormat</span><span class="p">.</span><span class="n">setMockInstance</span><span class="p">(</span><span class="n">job</span><span class="p">,</span> <span class="s">"mockInstance"</span><span class="p">);</span>
+</pre></div>
+
+
+<h2 id="wzxhzdk13wzxhzdk14-mini-accumulo-cluster"><a id=Mini_Accumulo_Cluster></a> Mini Accumulo Cluster</h2>
+<p>While the Mock Accumulo provides a lightweight implementation of the client API for unit testing, it is often necessary to write more realistic end-to-end integration tests that take advantage of the entire ecosystem. The Mini Accumulo Cluster makes this possible by configuring and starting Zookeeper, initializing Accumulo, and starting the Master as well as some Tablet Servers. It runs against the local filesystem instead of having to start up HDFS. </p>
+<p>To start it up, you will need to supply an empty directory and a root password as arguments: </p>
+<div class="codehilite"><pre><span class="n">File</span> <span class="n">tempDirectory</span> <span class="p">=</span> <span class="o">//</span> <span class="n">JUnit</span> <span class="n">and</span> <span class="n">Guava</span> <span class="n">supply</span> <span class="n">mechanisms</span> <span class="k">for</span> <span class="n">creating</span> <span class="n">temp</span> <span class="n">directories</span>
+<span class="n">MiniAccumuloCluster</span> <span class="n">accumulo</span> <span class="p">=</span> <span class="n">new</span> <span class="n">MiniAccumuloCluster</span><span class="p">(</span><span class="n">tempDirectory</span><span class="p">,</span> "<span class="n">password</span>"<span class="p">);</span>
+<span class="n">accumulo</span><span class="p">.</span><span class="n">start</span><span class="p">();</span>
+</pre></div>
+
+
+<p>Once we have our mini cluster running, we will want to interact with the Accumulo client API: </p>
+<div class="codehilite"><pre><span class="n">Instance</span> <span class="n">instance</span> <span class="p">=</span> <span class="n">new</span> <span class="n">ZooKeeperInstance</span><span class="p">(</span><span class="n">accumulo</span><span class="p">.</span><span class="n">getInstanceName</span><span class="p">(),</span> <span class="n">accumulo</span><span class="p">.</span><span class="n">getZooKeepers</span><span class="p">());</span>
+<span class="n">Connector</span> <span class="n">conn</span> <span class="p">=</span> <span class="n">instance</span><span class="p">.</span><span class="n">getConnector</span><span class="p">(</span>"<span class="n">root</span>"<span class="p">,</span> "<span class="n">password</span>"<span class="p">);</span>
+</pre></div>
+
+
+<p>Upon completion of our development code, we will want to shutdown our MiniAccumuloCluster: </p>
+<div class="codehilite"><pre><span class="n">accumulo</span><span class="p">.</span><span class="n">stop</span><span class="p">()</span>
+<span class="o">//</span> <span class="n">delete</span> <span class="n">your</span> <span class="n">temporary</span> <span class="n">folder</span>
+</pre></div>
+
+
+<hr />
+<p><strong> Next:</strong> <a href="Table_Configuration.html">Table Configuration</a> <strong> Up:</strong> <a href="accumulo_user_manual.html">Apache Accumulo User Manual Version 1.4</a> <strong> Previous:</strong> <a href="Writing_Accumulo_Clients.html">Writing Accumulo Clients</a> <strong> <a href="Contents.html">Contents</a></strong></p>
+ </div>
+
+ <div id="footer">
+ <a alt="Apache Software Foundation" href="http://www.apache.org">
+ <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif"/ width="100">
+ </a>
+ <div class="copyright">
+ <p>
+ Copyright © 2011-2013 The Apache Software Foundation, Licensed under
+ the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
+ <br />
+ Apache Accumulo, Accumulo, Apache, the Apache feather logo, and the Apache Accumulo
+ <br />
+ project logo are trademarks of the <a href="http://www.apache.org">Apache Software Foundation</a>.
+ </p>
+ </div>
+ </div>
+
+</body>
+</html>
Modified: websites/staging/accumulo/trunk/content/1.4/user_manual/Table_Configuration.html
==============================================================================
Binary files - no diff available.
Modified: websites/staging/accumulo/trunk/content/1.4/user_manual/Writing_Accumulo_Clients.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/user_manual/Writing_Accumulo_Clients.html (original)
+++ websites/staging/accumulo/trunk/content/1.4/user_manual/Writing_Accumulo_Clients.html Mon Mar 17 21:10:31 2014
@@ -91,17 +91,18 @@
<div id="content">
<h1 class="title">Apache Accumulo User Manual: Writing Accumulo Clients</h1>
- <p><strong> Next:</strong> <a href="Table_Configuration.html">Table Configuration</a> <strong> Up:</strong> <a href="accumulo_user_manual.html">Apache Accumulo User Manual Version 1.4</a> <strong> Previous:</strong> <a href="Accumulo_Shell.html">Accumulo Shell</a> <strong> <a href="Contents.html">Contents</a></strong> </p>
+ <p><strong> Next:</strong> <a href="Development_Clients.html">Development Clients</a> <strong> Up:</strong> <a href="accumulo_user_manual.html">Apache Accumulo User Manual Version 1.4</a> <strong> Previous:</strong> <a href="Accumulo_Shell.html">Accumulo Shell</a> <strong> <a href="Contents.html">Contents</a></strong> </p>
<p><a id=CHILD_LINKS></a><strong>Subsections</strong></p>
<ul>
<li><a href="Writing_Accumulo_Clients.html#Running_Client_Code">Running Client Code</a></li>
<li><a href="Writing_Accumulo_Clients.html#Connecting">Connecting</a></li>
<li><a href="Writing_Accumulo_Clients.html#Writing_Data">Writing Data</a></li>
<li><a href="Writing_Accumulo_Clients.html#Reading_Data">Reading Data</a></li>
+<li><a href="Writing_Accumulo_Clients.html#Proxy">Proxy</a></li>
</ul>
<hr />
-<h2 id="wzxhzdk8wzxhzdk9-writing-accumulo-clients"><a id=Writing_Accumulo_Clients></a> Writing Accumulo Clients</h2>
-<h2 id="wzxhzdk10wzxhzdk11-running-client-code"><a id=Running_Client_Code></a> Running Client Code</h2>
+<h2 id="wzxhzdk16wzxhzdk17-writing-accumulo-clients"><a id=Writing_Accumulo_Clients></a> Writing Accumulo Clients</h2>
+<h2 id="wzxhzdk18wzxhzdk19-running-client-code"><a id=Running_Client_Code></a> Running Client Code</h2>
<p>There are multiple ways to run Java code that uses Accumulo. Below is a list of the different ways to execute client code. </p>
<ul>
<li>using java executable </li>
@@ -115,7 +116,7 @@
<p>Another option for running your code is to put a jar file in $ACCUMULO_HOME/lib/ext. After doing this you can use the accumulo script to execute your code. For example if you create a jar containing the class com.foo.Client and placed that in lib/ext, then you could use the command $ACCUMULO_HOME/bin/accumulo com.foo.Client to execute your code. </p>
<p>If you are writing map reduce job that access Accumulo, then you can use the bin/tool.sh script to run those jobs. See the map reduce example. </p>
-<h2 id="wzxhzdk12wzxhzdk13-connecting"><a id=Connecting></a> Connecting</h2>
+<h2 id="wzxhzdk20wzxhzdk21-connecting"><a id=Connecting></a> Connecting</h2>
<p>All clients must first identify the Accumulo instance to which they will be communicating. Code to do this is as follows: </p>
<div class="codehilite"><pre><span class="n">String</span> <span class="n">instanceName</span> <span class="p">=</span> "<span class="n">myinstance</span>"<span class="p">;</span>
<span class="n">String</span> <span class="n">zooServers</span> <span class="p">=</span> "<span class="n">zooserver</span><span class="o">-</span><span class="n">one</span><span class="p">,</span><span class="n">zooserver</span><span class="o">-</span><span class="n">two</span>"
@@ -125,7 +126,7 @@
</pre></div>
-<h2 id="wzxhzdk14wzxhzdk15-writing-data"><a id=Writing_Data></a> Writing Data</h2>
+<h2 id="wzxhzdk22wzxhzdk23-writing-data"><a id=Writing_Data></a> Writing Data</h2>
<p>Data are written to Accumulo by creating Mutation objects that represent all the changes to the columns of a single row. The changes are made atomically in the TabletServer. Clients then add Mutations to a BatchWriter which submits them to the appropriate TabletServers. </p>
<p>Mutations can be created thus: </p>
<div class="codehilite"><pre><span class="n">Text</span> <span class="n">rowID</span> <span class="p">=</span> <span class="n">new</span> <span class="n">Text</span><span class="p">(</span>"<span class="n">row1</span>"<span class="p">);</span>
@@ -141,7 +142,7 @@
</pre></div>
-<h3 id="wzxhzdk16wzxhzdk17-batchwriter"><a id=BatchWriter></a> BatchWriter</h3>
+<h3 id="wzxhzdk24wzxhzdk25-batchwriter"><a id=BatchWriter></a> BatchWriter</h3>
<p>The BatchWriter is highly optimized to send Mutations to multiple TabletServers and automatically batches Mutations destined for the same TabletServer to amortize network overhead. Care must be taken to avoid changing the contents of any Object passed to the BatchWriter since it keeps objects in memory while batching. </p>
<p>Mutations are added to a BatchWriter thus: </p>
<div class="codehilite"><pre><span class="n">long</span> <span class="n">memBuf</span> <span class="p">=</span> 1000000<span class="n">L</span><span class="p">;</span> <span class="o">//</span> <span class="n">bytes</span> <span class="n">to</span> <span class="n">store</span> <span class="n">before</span> <span class="n">sending</span> <span class="n">a</span> <span class="n">batch</span>
@@ -159,9 +160,9 @@
<p>An example of using the batch writer can be found at <br />
accumulo/docs/examples/README.batch </p>
-<h2 id="wzxhzdk18wzxhzdk19-reading-data"><a id=Reading_Data></a> Reading Data</h2>
+<h2 id="wzxhzdk26wzxhzdk27-reading-data"><a id=Reading_Data></a> Reading Data</h2>
<p>Accumulo is optimized to quickly retrieve the value associated with a given key, and to efficiently return ranges of consecutive keys and their associated values. </p>
-<h3 id="wzxhzdk20wzxhzdk21-scanner"><a id=Scanner></a> Scanner</h3>
+<h3 id="wzxhzdk28wzxhzdk29-scanner"><a id=Scanner></a> Scanner</h3>
<p>To retrieve data, Clients use a Scanner, which provides acts like an Iterator over keys and values. Scanners can be configured to start and stop at particular keys, and to return a subset of the columns available. </p>
<div class="codehilite"><pre><span class="c1">// specify which visibilities we are allowed to see</span>
<span class="n">Authorizations</span> <span class="n">auths</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Authorizations</span><span class="p">(</span><span class="s">"public"</span><span class="p">);</span>
@@ -179,7 +180,7 @@ accumulo/docs/examples/README.batch </p>
</pre></div>
-<h3 id="wzxhzdk22wzxhzdk23-isolated-scanner"><a id=Isolated_Scanner></a> Isolated Scanner</h3>
+<h3 id="wzxhzdk30wzxhzdk31-isolated-scanner"><a id=Isolated_Scanner></a> Isolated Scanner</h3>
<p>Accumulo supports the ability to present an isolated view of rows when scanning. There are three possible ways that a row could change in accumulo : </p>
<ul>
<li>a mutation applied to a table </li>
@@ -190,7 +191,7 @@ accumulo/docs/examples/README.batch </p>
<p>The IsolatedScanner buffers rows on the client side so a large row will not crash a tablet server. By default rows are buffered in memory, but the user can easily supply their own buffer if they wish to buffer to disk when rows are large. </p>
<p>For an example, look at the following <br />
src/examples/src/main/java/org/apache/accumulo/examples/isolation/InterferenceTest.java</p>
-<h3 id="wzxhzdk24wzxhzdk25-batchscanner"><a id=BatchScanner></a> BatchScanner</h3>
+<h3 id="wzxhzdk32wzxhzdk33-batchscanner"><a id=BatchScanner></a> BatchScanner</h3>
<p>For some types of access, it is more efficient to retrieve several ranges simultaneously. This arises when accessing a set of rows that are not consecutive whose IDs have been retrieved from a secondary index, for example. </p>
<p>The BatchScanner is configured similarly to the Scanner; it can be configured to retrieve a subset of the columns available, but rather than passing a single Range, BatchScanners accept a set of Ranges. It is important to note that the keys returned by a BatchScanner are not in sorted order since the keys streamed are from multiple TabletServers in parallel. </p>
<div class="codehilite"><pre><span class="n">ArrayList</span><span class="o"><</span><span class="n">Range</span><span class="o">></span> <span class="n">ranges</span> <span class="p">=</span> <span class="n">new</span> <span class="n">ArrayList</span><span class="o"><</span><span class="n">Range</span><span class="o">></span><span class="p">();</span>
@@ -209,8 +210,82 @@ src/examples/src/main/java/org/apache/ac
<p>An example of the BatchScanner can be found at <br />
accumulo/docs/examples/README.batch </p>
+<h2 id="wzxhzdk34wzxhzdk35-proxy"><a id=Proxy></a> Proxy</h2>
+<p>The proxy API allows the interaction with Accumulo with languages other than Java. A proxy server is provided in the codebase and a client can further be generated. </p>
+<h3 id="wzxhzdk36wzxhzdk37-prequisites"><a id=Prequisites></a> Prequisites</h3>
+<p>The proxy server can live on any node in which the basic client API would work. That means it must be able to communicate with the Master, ZooKeepers, NameNode, and the Data nodes. A proxy client only needs the ability to communicate with the proxy server. </p>
+<h3 id="wzxhzdk38wzxhzdk39-configuration"><a id=Configuration></a> Configuration</h3>
+<p>The configuration options for the proxy server live inside of a properties file. At the very least, you need to supply the following properties: </p>
+<div class="codehilite"><pre><span class="n">protocolFactory</span><span class="p">=</span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">thrift</span><span class="p">.</span><span class="n">protocol</span><span class="p">.</span><span class="n">TCompactProtocol</span>$<span class="n">Factory</span>
+<span class="n">tokenClass</span><span class="p">=</span><span class="n">org</span><span class="p">.</span><span class="n">apache</span><span class="p">.</span><span class="n">accumulo</span><span class="p">.</span><span class="n">core</span><span class="p">.</span><span class="n">client</span><span class="p">.</span><span class="n">security</span><span class="p">.</span><span class="n">tokens</span><span class="p">.</span><span class="n">PasswordToken</span>
+<span class="n">port</span><span class="p">=</span>42424
+<span class="n">instance</span><span class="p">=</span><span class="n">test</span>
+<span class="n">zookeepers</span><span class="p">=</span><span class="n">localhost</span><span class="p">:</span>2181
+</pre></div>
+
+
+<p>You can find a sample configuration file in your distribution: </p>
+<div class="codehilite"><pre>$<span class="n">ACCUMULO_HOME</span><span class="o">/</span><span class="n">proxy</span><span class="o">/</span><span class="n">proxy</span><span class="p">.</span><span class="k">properties</span><span class="p">.</span>
+</pre></div>
+
+
+<p>This sample configuration file further demonstrates an ability to back the proxy server by MockAccumulo or the MiniAccumuloCluster. </p>
+<h3 id="wzxhzdk40wzxhzdk41-running-the-proxy-server"><a id=Running_the_Proxy_Server></a> Running the Proxy Server</h3>
+<p>After the properties file holding the configuration is created, the proxy server can be started using the following command in the Accumulo distribution (assuming your properties file is named config.properties): </p>
+<div class="codehilite"><pre>$<span class="n">ACCUMULO_HOME</span><span class="o">/</span><span class="n">bin</span><span class="o">/</span><span class="n">accumulo</span> <span class="n">proxy</span> <span class="o">-</span><span class="n">p</span> <span class="n">config</span><span class="p">.</span><span class="k">properties</span>
+</pre></div>
+
+
+<h3 id="wzxhzdk42wzxhzdk43-creating-a-proxy-client"><a id=Creating_a_Proxy_Client></a> Creating a Proxy Client</h3>
+<p>Aside from installing the Thrift compiler, you will also need the language-specific library for Thrift installed to generate client code in that language. Typically, your operating system's package manager will be able to automatically install these for you in an expected location such as /usr/lib/python/site-packages/thrift. </p>
+<p>You can find the thrift file for generating the client: </p>
+<div class="codehilite"><pre>$<span class="n">ACCUMULO_HOME</span><span class="o">/</span><span class="n">proxy</span><span class="o">/</span><span class="n">proxy</span><span class="p">.</span><span class="n">thrift</span><span class="p">.</span>
+</pre></div>
+
+
+<p>After a client is generated, the port specified in the configuration properties above will be used to connect to the server. </p>
+<h3 id="wzxhzdk44wzxhzdk45-using-a-proxy-client"><a id=Using_a_Proxy_Client></a> Using a Proxy Client</h3>
+<p>The following examples have been written in Java and the method signatures may be slightly different depending on the language specified when generating client with the Thrift compiler. After initiating a connection to the Proxy (see Apache Thrift's documentation for examples of connecting to a Thrift service), the methods on the proxy client will be available. The first thing to do is log in: </p>
+<div class="codehilite"><pre><span class="n">Map</span> <span class="n">password</span> <span class="p">=</span> <span class="n">new</span> <span class="n">HashMap</span><span class="o"><</span><span class="n">String</span><span class="p">,</span><span class="n">String</span><span class="o">></span><span class="p">();</span>
+<span class="n">password</span><span class="p">.</span><span class="n">put</span><span class="p">(</span>"<span class="n">password</span>"<span class="p">,</span> "<span class="n">secret</span>"<span class="p">);</span>
+<span class="n">ByteBuffer</span> <span class="n">token</span> <span class="p">=</span> <span class="n">client</span><span class="p">.</span><span class="n">login</span><span class="p">(</span>"<span class="n">root</span>"<span class="p">,</span> <span class="n">password</span><span class="p">);</span>
+</pre></div>
+
+
+<p>Once logged in, the token returned will be used for most subsequent calls to the client. Let's create a table, add some data, scan the table, and delete it. </p>
+<p>First, create a table. </p>
+<div class="codehilite"><pre><span class="n">client</span><span class="p">.</span><span class="n">createTable</span><span class="p">(</span><span class="n">token</span><span class="p">,</span> "<span class="n">myTable</span>"<span class="p">,</span> <span class="n">true</span><span class="p">,</span> <span class="n">TimeType</span><span class="p">.</span><span class="n">MILLIS</span><span class="p">);</span>
+</pre></div>
+
+
+<p>Next, add some data: </p>
+<div class="codehilite"><pre><span class="c1">// first, create a writer on the server</span>
+<span class="n">String</span> <span class="n">writer</span> <span class="o">=</span> <span class="n">client</span><span class="p">.</span><span class="n">createWriter</span><span class="p">(</span><span class="n">token</span><span class="p">,</span> <span class="s">"myTable"</span><span class="p">,</span> <span class="k">new</span> <span class="n">WriterOptions</span><span class="p">());</span>
+
+<span class="c1">// build column updates</span>
+<span class="n">Map</span><span class="o"><</span><span class="n">ByteBuffer</span><span class="p">,</span> <span class="n">List</span><span class="o"><</span><span class="n">ColumnUpdate</span><span class="o">></span> <span class="n">cells</span><span class="o">></span> <span class="n">cellsToUpdate</span> <span class="o">=</span> <span class="c1">//...</span>
+
+<span class="c1">// send updates to the server</span>
+<span class="n">client</span><span class="p">.</span><span class="n">updateAndFlush</span><span class="p">(</span><span class="n">writer</span><span class="p">,</span> <span class="s">"myTable"</span><span class="p">,</span> <span class="n">cellsToUpdate</span><span class="p">);</span>
+
+<span class="n">client</span><span class="p">.</span><span class="n">closeWriter</span><span class="p">(</span><span class="n">writer</span><span class="p">);</span>
+</pre></div>
+
+
+<p>Scan for the data and batch the return of the results on the server: </p>
+<div class="codehilite"><pre><span class="n">String</span> <span class="n">scanner</span> <span class="p">=</span> <span class="n">client</span><span class="p">.</span><span class="n">createScanner</span><span class="p">(</span><span class="n">token</span><span class="p">,</span> "<span class="n">myTable</span>"<span class="p">,</span> <span class="n">new</span> <span class="n">ScanOptions</span><span class="p">());</span>
+<span class="n">ScanResult</span> <span class="n">results</span> <span class="p">=</span> <span class="n">client</span><span class="p">.</span><span class="n">nextK</span><span class="p">(</span><span class="n">scanner</span><span class="p">,</span> 100<span class="p">);</span>
+
+<span class="k">for</span><span class="p">(</span><span class="n">KeyValue</span> <span class="n">keyValue</span> <span class="p">:</span> <span class="n">results</span><span class="p">.</span><span class="n">getResultsIterator</span><span class="p">())</span> <span class="p">{</span>
+ <span class="o">//</span> <span class="n">do</span> <span class="n">something</span> <span class="n">with</span> <span class="n">results</span>
+<span class="p">}</span>
+
+<span class="n">client</span><span class="p">.</span><span class="n">closeScanner</span><span class="p">(</span><span class="n">scanner</span><span class="p">);</span>
+</pre></div>
+
+
<hr />
-<p><strong> Next:</strong> <a href="Table_Configuration.html">Table Configuration</a> <strong> Up:</strong> <a href="accumulo_user_manual.html">Apache Accumulo User Manual Version 1.4</a> <strong> Previous:</strong> <a href="Accumulo_Shell.html">Accumulo Shell</a> <strong> <a href="Contents.html">Contents</a></strong></p>
+<p><strong> Next:</strong> <a href="Development_Clients.html">Development Clients</a> <strong> Up:</strong> <a href="accumulo_user_manual.html">Apache Accumulo User Manual Version 1.4</a> <strong> Previous:</strong> <a href="Accumulo_Shell.html">Accumulo Shell</a> <strong> <a href="Contents.html">Contents</a></strong></p>
</div>
<div id="footer">
Modified: websites/staging/accumulo/trunk/content/1.4/user_manual/accumulo_user_manual.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/user_manual/accumulo_user_manual.html (original)
+++ websites/staging/accumulo/trunk/content/1.4/user_manual/accumulo_user_manual.html Mon Mar 17 21:10:31 2014
@@ -102,6 +102,7 @@
<li><a href="Accumulo_Design.html">Accumulo Design</a></li>
<li><a href="Accumulo_Shell.html">Accumulo Shell</a></li>
<li><a href="Writing_Accumulo_Clients.html">Writing Accumulo Clients</a></li>
+<li><a href="Development_Clients.html">Development Clients</a></li>
<li><a href="Table_Configuration.html">Table Configuration</a></li>
<li><a href="Table_Design.html">Table Design</a></li>
<li><a href="High_Speed_Ingest.html">High-Speed Ingest</a></li>
Modified: websites/staging/accumulo/trunk/content/1.4/user_manual/index.html
==============================================================================
--- websites/staging/accumulo/trunk/content/1.4/user_manual/index.html (original)
+++ websites/staging/accumulo/trunk/content/1.4/user_manual/index.html Mon Mar 17 21:10:31 2014
@@ -102,6 +102,7 @@
<li><a href="Accumulo_Design.html">Accumulo Design</a></li>
<li><a href="Accumulo_Shell.html">Accumulo Shell</a></li>
<li><a href="Writing_Accumulo_Clients.html">Writing Accumulo Clients</a></li>
+<li><a href="Development_Clients.html">Development Clients</a></li>
<li><a href="Table_Configuration.html">Table Configuration</a></li>
<li><a href="Table_Design.html">Table Design</a></li>
<li><a href="High_Speed_Ingest.html">High-Speed Ingest</a></li>