You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/05/09 17:29:18 UTC
[40/50] [abbrv] git commit: ACCUMULO-1327 converted latex manual to
asciidoc
ACCUMULO-1327 converted latex manual to asciidoc
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/900d6abb
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/900d6abb
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/900d6abb
Branch: refs/heads/ACCUMULO-378
Commit: 900d6abb04dd1f5c87d16b55624055de6d11a6ab
Parents: cd04572
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed May 7 17:54:28 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed May 7 17:54:28 2014 -0700
----------------------------------------------------------------------
README | 2 +-
assemble/pom.xml | 2 +-
assemble/src/main/assemblies/component.xml | 2 +-
.../accumulo/core/conf/ConfigurationDocGen.java | 72 +-
.../core/conf/DefaultConfiguration.java | 4 +-
.../accumulo/core/conf/config-header.html | 2 -
.../apache/accumulo/core/conf/config-header.tex | 99 ---
.../apache/accumulo/core/conf/config-header.txt | 93 +++
docs/pom.xml | 67 +-
.../main/asciidoc/accumulo_user_manual.asciidoc | 62 ++
.../main/asciidoc/chapters/administration.txt | 396 +++++++++
docs/src/main/asciidoc/chapters/analytics.txt | 229 ++++++
docs/src/main/asciidoc/chapters/clients.txt | 320 ++++++++
docs/src/main/asciidoc/chapters/design.txt | 179 +++++
.../asciidoc/chapters/development_clients.txt | 107 +++
.../asciidoc/chapters/high_speed_ingest.txt | 125 +++
.../src/main/asciidoc/chapters/introduction.txt | 25 +
docs/src/main/asciidoc/chapters/multivolume.txt | 82 ++
docs/src/main/asciidoc/chapters/security.txt | 176 ++++
docs/src/main/asciidoc/chapters/shell.txt | 129 +++
.../asciidoc/chapters/table_configuration.txt | 635 +++++++++++++++
.../src/main/asciidoc/chapters/table_design.txt | 336 ++++++++
.../main/asciidoc/chapters/troubleshooting.txt | 758 ++++++++++++++++++
docs/src/main/asciidoc/images/accumulo-logo.png | Bin 0 -> 11427 bytes
.../main/asciidoc/images/data_distribution.png | Bin 0 -> 86937 bytes
.../main/asciidoc/images/failure_handling.png | Bin 0 -> 48905 bytes
.../accumulo_user_manual.tex | 55 --
.../chapters/administration.tex | 431 ----------
.../accumulo_user_manual/chapters/analytics.tex | 248 ------
.../accumulo_user_manual/chapters/clients.tex | 331 --------
.../accumulo_user_manual/chapters/design.tex | 186 -----
.../chapters/development_clients.tex | 109 ---
.../chapters/high_speed_ingest.tex | 133 ----
.../chapters/introduction.tex | 27 -
.../chapters/multivolume.tex | 85 --
.../accumulo_user_manual/chapters/security.tex | 179 -----
.../accumulo_user_manual/chapters/shell.tex | 138 ----
.../chapters/table_configuration.tex | 663 ----------------
.../chapters/table_design.tex | 343 --------
.../chapters/troubleshooting.tex | 794 -------------------
.../images/data_distribution.png | Bin 86937 -> 0 bytes
.../images/failure_handling.png | Bin 48905 -> 0 bytes
pom.xml | 12 +-
43 files changed, 3754 insertions(+), 3882 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/README
----------------------------------------------------------------------
diff --git a/README b/README
index a2d758b..e806dff 100644
--- a/README
+++ b/README
@@ -72,7 +72,7 @@ library into the $ACCUMULO_HOME/lib/native/map.
Building Documentation
-Use the following command to build the User Manual (docs/target/accumulo_user_manual.pdf)
+Use the following command to build the User Manual (docs/target/generated-docs/accumulo_user_manual.html)
and the configuration HTML page (docs/target/config.html)
mvn package -P docs -DskipTests
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/assemble/pom.xml
----------------------------------------------------------------------
diff --git a/assemble/pom.xml b/assemble/pom.xml
index d30c265..bbd409b 100644
--- a/assemble/pom.xml
+++ b/assemble/pom.xml
@@ -227,7 +227,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-docs</artifactId>
<classifier>user-manual</classifier>
- <type>pdf</type>
+ <type>html</type>
</dependency>
</dependencies>
</profile>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/assemble/src/main/assemblies/component.xml
----------------------------------------------------------------------
diff --git a/assemble/src/main/assemblies/component.xml b/assemble/src/main/assemblies/component.xml
index 747444b..5ef9914 100644
--- a/assemble/src/main/assemblies/component.xml
+++ b/assemble/src/main/assemblies/component.xml
@@ -47,7 +47,7 @@
<outputFileNameMapping>${artifactId}_user_manual.${artifact.extension}</outputFileNameMapping>
<useTransitiveDependencies>false</useTransitiveDependencies>
<includes>
- <include>${groupId}:${artifactId}-docs:pdf:user-manual</include>
+ <include>${groupId}:${artifactId}-docs:html:user-manual</include>
</includes>
</dependencySet>
</dependencySets>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index 8ed5506..6cf9c77 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@ -39,6 +39,7 @@ class ConfigurationDocGen {
pageHeader();
beginSection("Available Properties");
+ propertyQuickLinks();
for (Property prefix : prefixes) {
if (!prefix.isExperimental()) {
prefixSection(prefix);
@@ -75,6 +76,8 @@ class ConfigurationDocGen {
abstract void propertyTypeDescriptions();
+ abstract void propertyQuickLinks();
+
abstract String sanitize(String str);
}
@@ -130,8 +133,7 @@ class ConfigurationDocGen {
}
@Override
- void pageHeader() {
- super.pageHeader();
+ void propertyQuickLinks() {
doc.println("<p>Jump to: ");
String delimiter = "";
for (Property prefix : prefixes) {
@@ -210,22 +212,37 @@ class ConfigurationDocGen {
}
- private class LaTeX extends Format {
+ private class Asciidoc extends Format {
@Override
void beginSection(String section) {
- doc.println("\\section{" + section + "}");
+ doc.println("=== " + section);
}
@Override
String getExt() {
- return "tex";
+ return "txt";
+ }
+
+ @Override
+ void propertyQuickLinks() {
+ doc.println("Jump to: ");
+ String delimiter = "";
+ for (Property prefix : prefixes) {
+ if (!prefix.isExperimental()) {
+ doc.print(delimiter + "<<" + prefix.name() + ">>");
+ delimiter = " | ";
+ }
+ }
+ doc.println();
+ doc.println();
}
@Override
void prefixSection(Property prefix) {
boolean depr = prefix.isDeprecated();
- doc.println("\\subsection{" + prefix.getKey() + "*}" + (depr ? " (Deprecated)" : ""));
- doc.println((depr ? "\\sout{\\textit{Deprecated.} " : "") + sanitize(prefix.getDescription()) + (depr ? "}" : ""));
+ doc.println("[[" + prefix.name() + "]]");
+ doc.println("==== " + prefix.getKey() + "*" + (depr ? " (Deprecated)" : ""));
+ doc.println(strike((depr ? "_Deprecated._ " : "") + sanitize(prefix.getDescription()), depr));
doc.println();
}
@@ -233,18 +250,28 @@ class ConfigurationDocGen {
void property(Property prefix, Property prop) {
boolean depr = prefix.isDeprecated() || prop.isDeprecated();
if (prop.getKey().startsWith(prefix.getKey())) {
- doc.println("\\subsubsection{" + prop.getKey() + "}");
- doc.println(strike((depr ? "\\textit{Deprecated.} " : "") + sanitize(prop.getDescription()), depr));
+ doc.println("===== " + prop.getKey());
+ doc.println(strike((depr ? "_Deprecated._ " : "") + sanitize(prop.getDescription()), depr));
doc.println();
- doc.println(strike("\\textit{Type:} " + prop.getType().name() + "\\\\", depr));
- doc.println(strike("\\textit{ZooKeeper Mutable:} " + isZooKeeperMutable(prop) + "\\\\", depr));
- doc.println(strike("\\textit{Default Value:} " + sanitize(prop.getRawDefaultValue()), depr));
+ doc.println(strike("_Type:_ " + prop.getType().name(), depr) + " +");
+ doc.println(strike("_Zookeeper Mutable:_ " + isZooKeeperMutable(prop), depr) + " +");
+ String defaultValue = sanitize(prop.getRawDefaultValue()).trim();
+ if (defaultValue.length() == 0) {
+ // need a placeholder or the asciidoc line break won't work
+ defaultValue = strike("_Default Value:_ _empty_", depr);
+ } else if (defaultValue.contains("\n")) {
+ // deal with multi-line values, skip strikethrough of value
+ defaultValue = strike("_Default Value:_ ", depr) + "\n----\n" + defaultValue + "\n----\n";
+ } else {
+ defaultValue = strike("_Default Value:_ " + "`" + defaultValue + "`", depr);
+ }
+ doc.println(defaultValue);
doc.println();
}
}
private String strike(String s, boolean isDeprecated) {
- return (isDeprecated ? "\\sout{" : "") + s + (isDeprecated ? "}" : "");
+ return (isDeprecated ? "[line-through]#" : "") + s + (isDeprecated ? "#" : "");
}
@Override
@@ -252,7 +279,7 @@ class ConfigurationDocGen {
for (PropertyType type : PropertyType.values()) {
if (type == PropertyType.PREFIX)
continue;
- doc.println("\\subsection{" + sanitize(type.toString()) + "}");
+ doc.println("==== " + sanitize(type.toString()));
doc.println(sanitize(type.getFormatDescription()));
doc.println();
}
@@ -260,19 +287,6 @@ class ConfigurationDocGen {
@Override
String sanitize(String str) {
- str = str.replace("\\", "\\textbackslash{}");
- str = str.replace("~", "\\textasciitilde{}");
- str = str.replace("^", "\\textasciicircum");
-
- str = str.replace("&", "\\&");
- str = str.replace("%", "\\%");
- str = str.replace("$", "\\$");
- str = str.replace("#", "\\#");
- str = str.replace("_", "\\_");
- str = str.replace("{", "\\{");
- str = str.replace("}", "\\}");
-
- str = str.replaceAll("(?:\r\n|\r|\n)", "\\\\\\\\\n");
return str;
}
}
@@ -334,8 +348,8 @@ class ConfigurationDocGen {
new HTML().generate();
}
- void generateLaTeX() {
- new LaTeX().generate();
+ void generateAsciidoc() {
+ new Asciidoc().generate();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
index b14d924..f3f43b9 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
@@ -79,8 +79,8 @@ public class DefaultConfiguration extends AccumuloConfiguration {
public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
if (args.length == 2 && args[0].equals("--generate-html")) {
new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateHtml();
- } else if (args.length == 2 && args[0].equals("--generate-latex")) {
- new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateLaTeX();
+ } else if (args.length == 2 && args[0].equals("--generate-asciidoc")) {
+ new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateAsciidoc();
} else {
throw new IllegalArgumentException("Usage: " + DefaultConfiguration.class.getName() + " --generate-html <filename> | --generate-latex <filename>");
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html
index 8270ad2..9c770b1 100644
--- a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html
+++ b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html
@@ -86,5 +86,3 @@
default | table.failures.ignore ..................... | false
</pre>
- <h1>Configuration Properties</h1>
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/resources/org/apache/accumulo/core/conf/config-header.tex
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.tex b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.tex
deleted file mode 100644
index a1e1dc0..0000000
--- a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.tex
+++ /dev/null
@@ -1,99 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Configuration Management}
-\label{app:config}
-
-\section{Configuration Overview}
-
-All accumulo properties have a default value in the source code. Properties can also be set
-in accumulo-site.xml and in zookeeper on per-table or system-wide basis. If properties are set in more than one location,
-accumulo will choose the property with the highest precedence. This order of precedence is described
-below (from highest to lowest):
-
-\subsection{Zookeeper table properties}
-Table properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. While table properties take precedent over system properties, both will override properties set in accumulo-site.xml
-
-Table properties consist of all properties with the table.* prefix. Table properties are configured on a per-table basis using the following shell commmand:
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- config -t TABLE -s PROPERTY=VALUE
-\end{verbatim}\endgroup
-
-\subsection{Zookeeper system properties}
-
-System properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. System properties consist of all properties with a `yes' in the `Zookeeper Mutable' column in the table below. They are set with the following shell command:
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- config -s PROPERTY=VALUE
-\end{verbatim}\endgroup
-
-If a table.* property is set using this method, the value will apply to all tables except those configured on per-table basis (which have higher precedence).
-
-While most system properties take effect immediately, some require a restart of the process which is indicated in `Zookeeper Mutable'.
-
-\subsection{accumulo-site.xml}
-
-Accumulo processes (master, tserver, etc) read their local accumulo-site.xml on start up. Therefore, changes made to accumulo-site.xml must rsynced across the cluster and processes must be restarted to apply changes.
-
-Certain properties (indicated by a `no' in `Zookeeper Mutable') cannot be set in zookeeper and only set in this file. The accumulo-site.xml also allows you to configure tablet servers with different settings.
-
-\subsection{Default Values}
-
-All properties have a default value in the source code. This value has the lowest precedence and is overriden if set in accumulo-site.xml or zookeeper.
-
-While the default value is usually optimal, there are cases where a change can increase query and ingest performance.
-
-\section{Configuration in the Shell}
-
-The `config' command in the shell allows you to view the current system configuration. You can also use the `-t' option to view a table's configuration as below:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-
- $ ./bin/accumulo shell -u root
- Enter current password for 'root'@'ac14': ******
-
- Shell - Apache Accumulo Interactive Shell
- -
- - version: 1.5.0
- - instance name: ac14
- - instance id: 4f48fa03-f692-43ce-ae03-94c9ea8b7181
- -
- - type 'help' for a list of available commands
- -
- root@ac13> config -t foo
- ---------+---------------------------------------------+------------------------------------------------------
- SCOPE | NAME | VALUE
- ---------+---------------------------------------------+------------------------------------------------------
- default | table.balancer ............................ | org.apache.accumulo.server.master.balancer.DefaultLoadBalancer
- default | table.bloom.enabled ....................... | false
- default | table.bloom.error.rate .................... | 0.5%
- default | table.bloom.hash.type ..................... | murmur
- default | table.bloom.key.functor ................... | org.apache.accumulo.core.file.keyfunctor.RowFunctor
- default | table.bloom.load.threshold ................ | 1
- default | table.bloom.size .......................... | 1048576
- default | table.cache.block.enable .................. | false
- default | table.cache.index.enable .................. | false
- default | table.compaction.major.everything.at ...... | 19700101000000GMT
- default | table.compaction.major.everything.idle .... | 1h
- default | table.compaction.major.ratio .............. | 1.3
- site | @override .............................. | 1.4
- system | @override .............................. | 1.5
- table | @override .............................. | 1.6
- default | table.compaction.minor.idle ............... | 5m
- default | table.compaction.minor.logs.threshold ..... | 3
- default | table.failures.ignore ..................... | false
-
-\end{verbatim}\endgroup
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/resources/org/apache/accumulo/core/conf/config-header.txt
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.txt b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.txt
new file mode 100644
index 0000000..d278882
--- /dev/null
+++ b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.txt
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+[[configuration]]
+== Configuration Management
+
+=== Configuration Overview
+
+All accumulo properties have a default value in the source code. Properties can also be set
+in accumulo-site.xml and in zookeeper on per-table or system-wide basis. If properties are set in more than one location,
+accumulo will choose the property with the highest precedence. This order of precedence is described
+below (from highest to lowest):
+
+==== Zookeeper table properties
+Table properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. While table properties take precedent over system properties, both will override properties set in accumulo-site.xml
+
+Table properties consist of all properties with the table.* prefix. Table properties are configured on a per-table basis using the following shell commmand:
+
+ config -t TABLE -s PROPERTY=VALUE
+
+==== Zookeeper system properties
+
+System properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. System properties consist of all properties with a `yes` in the _Zookeeper Mutable_ column in the table below. They are set with the following shell command:
+
+ config -s PROPERTY=VALUE
+
+If a table.* property is set using this method, the value will apply to all tables except those configured on per-table basis (which have higher precedence).
+
+While most system properties take effect immediately, some require a restart of the process which is indicated in _Zookeeper Mutable_.
+
+==== accumulo-site.xml
+
+Accumulo processes (master, tserver, etc) read their local accumulo-site.xml on start up. Therefore, changes made to accumulo-site.xml must rsynced across the cluster and processes must be restarted to apply changes.
+
+Certain properties (indicated by a `no` in _Zookeeper Mutable_) cannot be set in zookeeper and only set in this file. The accumulo-site.xml also allows you to configure tablet servers with different settings.
+
+==== Default Values
+
+All properties have a default value in the source code. This value has the lowest precedence and is overriden if set in accumulo-site.xml or zookeeper.
+
+While the default value is usually optimal, there are cases where a change can increase query and ingest performance.
+
+=== Configuration in the Shell
+
+The `config` command in the shell allows you to view the current system configuration. You can also use the `-t` option to view a table's configuration as below:
+
+----
+$ ./bin/accumulo shell -u root
+Enter current password for 'root'@'accumulo-instance': ******
+
+Shell - Apache Accumulo Interactive Shell
+-
+- version: 1.6.0
+- instance name: accumulo-instance
+- instance id: 4f48fa03-f692-43ce-ae03-94c9ea8b7181
+-
+- type 'help' for a list of available commands
+-
+root@accumulo-instance> config -t foo
+---------+---------------------------------------------+------------------------------------------------------
+SCOPE | NAME | VALUE
+---------+---------------------------------------------+------------------------------------------------------
+default | table.balancer ............................ | org.apache.accumulo.server.master.balancer.DefaultLoadBalancer
+default | table.bloom.enabled ....................... | false
+default | table.bloom.error.rate .................... | 0.5%
+default | table.bloom.hash.type ..................... | murmur
+default | table.bloom.key.functor ................... | org.apache.accumulo.core.file.keyfunctor.RowFunctor
+default | table.bloom.load.threshold ................ | 1
+default | table.bloom.size .......................... | 1048576
+default | table.cache.block.enable .................. | false
+default | table.cache.index.enable .................. | false
+default | table.compaction.major.everything.at ...... | 19700101000000GMT
+default | table.compaction.major.everything.idle .... | 1h
+default | table.compaction.major.ratio .............. | 1.3
+site | @override .............................. | 1.4
+system | @override .............................. | 1.5
+table | @override .............................. | 1.6
+default | table.compaction.minor.idle ............... | 5m
+default | table.compaction.minor.logs.threshold ..... | 3
+default | table.failures.ignore ..................... | false
+----
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index c438eb0..4cc75a9 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -38,6 +38,48 @@
<build>
<plugins>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-asciidoc</id>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <outputDirectory>${project.build.directory}/asciidoc</outputDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/asciidoc</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.asciidoctor</groupId>
+ <artifactId>asciidoctor-maven-plugin</artifactId>
+ <configuration>
+ <backend>html</backend>
+ <doctype>book</doctype>
+ <embedAssets>true</embedAssets>
+ <imagesDir>${project.build.directory}/asciidoc/images</imagesDir>
+ <sourceDirectory>${project.build.directory}/asciidoc</sourceDirectory>
+ <sourceHighlighter>highlightjs</sourceHighlighter>
+ </configuration>
+ <executions>
+ <execution>
+ <id>output-html</id>
+ <goals>
+ <goal>process-asciidoc</goal>
+ </goals>
+ <phase>prepare-package</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<executions>
@@ -51,7 +93,7 @@
<executable>mkdir</executable>
<arguments>
<argument>-p</argument>
- <argument>${project.build.directory}/latex/accumulo_user_manual/appendices</argument>
+ <argument>${project.build.directory}/asciidoc/appendices</argument>
</arguments>
</configuration>
</execution>
@@ -80,8 +122,8 @@
<mainClass>org.apache.accumulo.core.conf.DefaultConfiguration</mainClass>
<classpathScope>compile</classpathScope>
<arguments>
- <argument>--generate-latex</argument>
- <argument>${project.build.directory}/latex/accumulo_user_manual/appendices/config.tex</argument>
+ <argument>--generate-asciidoc</argument>
+ <argument>${project.build.directory}/asciidoc/appendices/config.txt</argument>
</arguments>
</configuration>
</execution>
@@ -89,31 +131,18 @@
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
- <artifactId>latex-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>build-pdf-manuals</id>
- <goals>
- <goal>latex</goal>
- </goals>
- <phase>prepare-package</phase>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
- <id>attach-user-manual-pdf</id>
+ <id>attach-user-manual-html</id>
<goals>
<goal>attach-artifact</goal>
</goals>
<configuration>
<artifacts>
<artifact>
- <file>${project.build.directory}/accumulo_user_manual.pdf</file>
- <type>pdf</type>
+ <file>${project.build.directory}/generated-docs/accumulo_user_manual.html</file>
+ <type>html</type>
<classifier>user-manual</classifier>
</artifact>
</artifacts>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/accumulo_user_manual.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/accumulo_user_manual.asciidoc b/docs/src/main/asciidoc/accumulo_user_manual.asciidoc
new file mode 100644
index 0000000..e3f9d4b
--- /dev/null
+++ b/docs/src/main/asciidoc/accumulo_user_manual.asciidoc
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+Apache Accumulo User Manual Version 1.6
+=======================================
+:author: Apache Accumulo Project
+:email: dev@accumulo.apache.org
+//:theme: flask
+:toc2:
+:toclevels: 4
+:toc-title: Apache Accumulo 1.6
+:numbered:
+//:max-width: 50em
+:website: http://accumulo.apache.org/
+
+image::accumulo-logo.png[]
+
+Copyright © 2011-2014 The Apache Software Foundation, Licensed under the Apache
+License, Version 2.0. Apache Accumulo, Accumulo, Apache, and the Apache
+Accumulo project logo are trademarks of the Apache Software Foundation.
+
+include::chapters/introduction.txt[]
+
+include::chapters/design.txt[]
+
+include::chapters/shell.txt[]
+
+include::chapters/clients.txt[]
+
+include::chapters/development_clients.txt[]
+
+include::chapters/table_configuration.txt[]
+
+include::chapters/table_design.txt[]
+
+include::chapters/high_speed_ingest.txt[]
+
+include::chapters/analytics.txt[]
+
+include::chapters/security.txt[]
+
+include::chapters/administration.txt[]
+
+include::chapters/multivolume.txt[]
+
+include::chapters/troubleshooting.txt[]
+
+[appendix]
+include::appendices/config.txt[]
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/administration.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/administration.txt b/docs/src/main/asciidoc/chapters/administration.txt
new file mode 100644
index 0000000..4dbcd1b
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/administration.txt
@@ -0,0 +1,396 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Administration
+
+=== Hardware
+
+Because we are running essentially two or three systems simultaneously layered
+across the cluster: HDFS, Accumulo and MapReduce, it is typical for hardware to
+consist of 4 to 8 cores, and 8 to 32 GB RAM. This is so each running process can have
+at least one core and 2 - 4 GB each.
+
+One core running HDFS can typically keep 2 to 4 disks busy, so each machine may
+typically have as little as 2 x 300GB disks and as much as 4 x 1TB or 2TB disks.
+
+It is possible to do with less than this, such as with 1u servers with 2 cores and 4GB
+each, but in this case it is recommended to only run up to two processes per
+machine -- i.e. DataNode and TabletServer or DataNode and MapReduce worker but
+not all three. The constraint here is having enough available heap space for all the
+processes on a machine.
+
+=== Network
+
+Accumulo communicates via remote procedure calls over TCP/IP for both passing
+data and control messages. In addition, Accumulo uses HDFS clients to
+communicate with HDFS. To achieve good ingest and query performance, sufficient
+network bandwidth must be available between any two machines.
+
+In addition to needing access to ports associated with HDFS and ZooKeeper, Accumulo will
+use the following default ports. Please make sure that they are open, or change
+their value in conf/accumulo-site.xml.
+
+.Accumulo default ports
+[width="75%",cols=">,^2,^2"]
+[options="header"]
+|====
+|Port | Description | Property Name
+|4445 | Shutdown Port (Accumulo MiniCluster) | n/a
+|4560 | Accumulo monitor (for centralized log display) | monitor.port.log4j
+|9997 | Tablet Server | tserver.port.client
+|9999 | Master Server | master.port.client
+|12234 | Accumulo Tracer | trace.port.client
+|42424 | Accumulo Proxy Server | n/a
+|50091 | Accumulo GC | gc.port.client
+|50095 | Accumulo HTTP monitor | monitor.port.client
+|====
+
+In addition, the user can provide +0+ and an ephemeral port will be chosen instead. This
+ephemeral port is likely to be unique and not already bound. Thus, configuring ports to
+use +0+ instead of an explicit value, should, in most cases, work around any issues of
+running multiple distinct Accumulo instances (or any other process which tries to use the
+same default ports) on the same hardware.
+
+=== Installation
+Choose a directory for the Accumulo installation. This directory will be referenced
+by the environment variable +$ACCUMULO_HOME+. Run the following:
+
+ $ tar xzf accumulo-1.6.0-bin.tar.gz # unpack to subdirectory
+ $ mv accumulo-1.6.0 $ACCUMULO_HOME # move to desired location
+
+Repeat this step at each machine within the cluster. Usually all machines have the
+same +$ACCUMULO_HOME+.
+
+=== Dependencies
+Accumulo requires HDFS and ZooKeeper to be configured and running
+before starting. Password-less SSH should be configured between at least the
+Accumulo master and TabletServer machines. It is also a good idea to run Network
+Time Protocol (NTP) within the cluster to ensure nodes' clocks don't get too out of
+sync, which can cause problems with automatically timestamped data.
+
+=== Configuration
+
+Accumulo is configured by editing several Shell and XML files found in
++$ACCUMULO_HOME/conf+. The structure closely resembles Hadoop's configuration
+files.
+
+==== Edit conf/accumulo-env.sh
+
+Accumulo needs to know where to find the software it depends on. Edit accumulo-env.sh
+and specify the following:
+
+. Enter the location of the installation directory of Accumulo for +$ACCUMULO_HOME+
+. Enter your system's Java home for +$JAVA_HOME+
+. Enter the location of Hadoop for +$HADOOP_PREFIX+
+. Choose a location for Accumulo logs and enter it for +$ACCUMULO_LOG_DIR+
+. Enter the location of ZooKeeper for +$ZOOKEEPER_HOME+
+
+By default Accumulo TabletServers are set to use 1GB of memory. You may change
+this by altering the value of +$ACCUMULO_TSERVER_OPTS+. Note the syntax is that of
+the Java JVM command line options. This value should be less than the physical
+memory of the machines running TabletServers.
+
+There are similar options for the master's memory usage and the garbage collector
+process. Reduce these if they exceed the physical RAM of your hardware and
+increase them, within the bounds of the physical RAM, if a process fails because of
+insufficient memory.
+
+Note that you will be specifying the Java heap space in accumulo-env.sh. You should
+make sure that the total heap space used for the Accumulo tserver and the Hadoop
+DataNode and TaskTracker is less than the available memory on each slave node in
+the cluster. On large clusters, it is recommended that the Accumulo master, Hadoop
+NameNode, secondary NameNode, and Hadoop JobTracker all be run on separate
+machines to allow them to use more heap space. If you are running these on the
+same machine on a small cluster, likewise make sure their heap space settings fit
+within the available memory.
+
+==== Native Map
+
+The tablet server uses a data structure called a MemTable to store sorted key/value
+pairs in memory when they are first received from the client. When a minor compaction
+occurs, this data structure is written to HDFS. The MemTable will default to using
+memory in the JVM but a JNI version, called the native map, can be used to significantly
+speed up performance by utilizing the memory space of the native operating system. The
+native map also avoids the performance implications brought on by garbage collection
+in the JVM by causing it to pause much less frequently.
+
+32-bit and 64-bit Linux versions of the native map ship with the Accumulo dist package.
+For other operating systems, the native map can be built from the codebase in two ways-
+from maven or from the Makefile.
+
+. Build from maven using the following command: +mvn clean package -Pnative+.
+. Build from the c++ source by running +make+ in the `$ACCUMULO_HOME/server/src/main/c++` directory.
+
+After building the native map from the source, you will find the artifact in
++$ACCUMULO_HOME/lib/native+. Upon starting up, the tablet server will look
+in this directory for the map library. If the file is renamed or moved from its
+target directory, the tablet server may not be able to find it.
+
+==== Cluster Specification
+
+On the machine that will serve as the Accumulo master:
+
+. Write the IP address or domain name of the Accumulo Master to the +$ACCUMULO_HOME/conf/masters+ file.
+. Write the IP addresses or domain name of the machines that will be TabletServers in +$ACCUMULO_HOME/conf/slaves+, one per line.
+
+Note that if using domain names rather than IP addresses, DNS must be configured
+properly for all machines participating in the cluster. DNS can be a confusing source
+of errors.
+
+==== Accumulo Settings
+Specify appropriate values for the following settings in
++$ACCUMULO_HOME/conf/accumulo-site.xml+ :
+
+[source,xml]
+<property>
+ <name>instance.zookeeper.host</name>
+ <value>zooserver-one:2181,zooserver-two:2181</value>
+ <description>list of zookeeper servers</description>
+</property>
+
+This enables Accumulo to find ZooKeeper. Accumulo uses ZooKeeper to coordinate
+settings between processes and helps finalize TabletServer failure.
+
+[source,xml]
+<property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+</property>
+
+The instance needs a secret to enable secure communication between servers. Configure your
+secret and make sure that the +accumulo-site.xml+ file is not readable to other users.
+
+Some settings can be modified via the Accumulo shell and take effect immediately, but
+some settings require a process restart to take effect. See the configuration documentation
+(available in the docs directory of the tarball and in <<configuration>>) for details.
+
+==== Deploy Configuration
+
+Copy the masters, slaves, accumulo-env.sh, and if necessary, accumulo-site.xml
+from the +$ACCUMULO_HOME/conf/+ directory on the master to all the machines
+specified in the slaves file.
+
+=== Initialization
+
+Accumulo must be initialized to create the structures it uses internally to locate
+data across the cluster. HDFS is required to be configured and running before
+Accumulo can be initialized.
+
+Once HDFS is started, initialization can be performed by executing
++$ACCUMULO_HOME/bin/accumulo init+ . This script will prompt for a name
+for this instance of Accumulo. The instance name is used to identify a set of tables
+and instance-specific settings. The script will then write some information into
+HDFS so Accumulo can start properly.
+
+The initialization script will prompt you to set a root password. Once Accumulo is
+initialized it can be started.
+
+=== Running
+
+==== Starting Accumulo
+
+Make sure Hadoop is configured on all of the machines in the cluster, including
+access to a shared HDFS instance. Make sure HDFS and ZooKeeper are running.
+Make sure ZooKeeper is configured and running on at least one machine in the
+cluster.
+Start Accumulo using the +bin/start-all.sh+ script.
+
+To verify that Accumulo is running, check the Status page as described under
+_Monitoring_. In addition, the Shell can provide some information about the status of
+tables via reading the metadata tables.
+
+==== Stopping Accumulo
+
+To shutdown cleanly, run +bin/stop-all.sh+ and the master will orchestrate the
+shutdown of all the tablet servers. Shutdown waits for all minor compactions to finish, so it may
+take some time for particular configurations.
+
+==== Adding a Node
+
+Update your +$ACCUMULO_HOME/conf/slaves+ (or +$ACCUMULO_CONF_DIR/slaves+) file to account for the addition.
+
+ $ACCUMULO_HOME/bin/accumulo admin start <host(s)> {<host> ...}
+
+Alternatively, you can ssh to each of the hosts you want to add and run:
+
+ $ACCUMULO_HOME/bin/start-here.sh
+
+Make sure the host in question has the new configuration, or else the tablet
+server won't start; at a minimum this needs to be on the host(s) being added,
+but in practice it's good to ensure consistent configuration across all nodes.
+
+==== Decomissioning a Node
+
+If you need to take a node out of operation, you can trigger a graceful shutdown of a tablet
+server. Accumulo will automatically rebalance the tablets across the available tablet servers.
+
+ $ACCUMULO_HOME/bin/accumulo admin stop <host(s)> {<host> ...}
+
+Alternatively, you can ssh to each of the hosts you want to remove and run:
+
+ $ACCUMULO_HOME/bin/stop-here.sh
+
+Be sure to update your +$ACCUMULO_HOME/conf/slaves+ (or +$ACCUMULO_CONF_DIR/slaves+) file to
+account for the removal of these hosts. Bear in mind that the monitor will not re-read the
+slaves file automatically, so it will report the decomissioned servers as down; it's
+recommended that you restart the monitor so that the node list is up to date.
+
+=== Monitoring
+
+The Accumulo Master provides an interface for monitoring the status and health of
+Accumulo components. This interface can be accessed by pointing a web browser to
++http://accumulomaster:50095/status+
+
+=== Tracing
+It can be difficult to determine why some operations are taking longer
+than expected. For example, you may be looking up items with very low
+latency, but sometimes the lookups take much longer. Determining the
+cause of the delay is difficult because the system is distributed, and
+the typical lookup is fast.
+
+Accumulo has been instrumented to record the time that various
+operations take when tracing is turned on. The fact that tracing is
+enabled follows all the requests made on behalf of the user throughout
+the distributed infrastructure of accumulo, and across all threads of
+execution.
+
+These time spans will be inserted into the +trace+ table in
+Accumulo. You can browse recent traces from the Accumulo monitor
+page. You can also read the +trace+ table directly like any
+other table.
+
+The design of Accumulo's distributed tracing follows that of
+http://research.google.com/pubs/pub36356.html[Google's Dapper].
+
+==== Tracers
+To collect traces, Accumulo needs at least one server listed in
+ +$ACCUMULO_HOME/conf/tracers+. The server collects traces
+from clients and writes them to the +trace+ table. The Accumulo
+user that the tracer connects to Accumulo with can be configured with
+the following properties
+
+ trace.user
+ trace.token.property.password
+
+==== Instrumenting a Client
+Tracing can be used to measure a client operation, such as a scan, as
+the operation traverses the distributed system. To enable tracing for
+your application call
+
+[source,java]
+DistributedTrace.enable(instance, new ZooReader(instance), hostname, "myApplication");
+
+Once tracing has been enabled, a client can wrap an operation in a trace.
+
+[source,java]
+Trace.on("Client Scan");
+BatchScanner scanner = conn.createBatchScanner(...);
+// Configure your scanner
+for (Entry entry : scanner) {
+}
+Trace.off();
+
+Additionally, the user can create additional Spans within a Trace.
+
+[source,java]
+Trace.on("Client Update");
+...
+Span readSpan = Trace.start("Read");
+...
+readSpan.stop();
+...
+Span writeSpan = Trace.start("Write");
+...
+writeSpan.stop();
+Trace.off();
+
+Like Dapper, Accumulo tracing supports user defined annotations to associate additional data with a Trace.
+
+[source,java]
+...
+int numberOfEntriesRead = 0;
+Span readSpan = Trace.start("Read");
+// Do the read, update the counter
+...
+readSpan.data("Number of Entries Read", String.valueOf(numberOfEntriesRead));
+
+Some client operations may have a high volume within your
+application. As such, you may wish to only sample a percentage of
+operations for tracing. As seen below, the CountSampler can be used to
+help enable tracing for 1-in-1000 operations
+
+[source,java]
+Sampler sampler = new CountSampler(1000);
+...
+if (sampler.next()) {
+ Trace.on("Read");
+}
+...
+Trace.offNoFlush();
+
+It should be noted that it is safe to turn off tracing even if it
+isn't currently active. The +Trace.offNoFlush()+ should be used if the
+user does not wish to have +Trace.off()+ block while flushing trace
+data.
+
+==== Viewing Collected Traces
+To view collected traces, use the "Recent Traces" link on the Monitor
+UI. You can also programmatically access and print traces using the
++TraceDump+ class.
+
+==== Tracing from the Shell
+You can enable tracing for operations run from the shell by using the
++trace on+ and +trace off+ commands.
+
+----
+root@test test> trace on
+
+root@test test> scan
+a b:c [] d
+
+root@test test> trace off
+Waiting for trace information
+Waiting for trace information
+Trace started at 2013/08/26 13:24:08.332
+Time Start Service@Location Name
+ 3628+0 shell@localhost shell:root
+ 8+1690 shell@localhost scan
+ 7+1691 shell@localhost scan:location
+ 6+1692 tserver@localhost startScan
+ 5+1692 tserver@localhost tablet read ahead 6
+----
+
+=== Logging
+Accumulo processes each write to a set of log files. By default these are found under
++$ACCUMULO/logs/+.
+
+=== Recovery
+
+In the event of TabletServer failure or error on shutting Accumulo down, some
+mutations may not have been minor compacted to HDFS properly. In this case,
+Accumulo will automatically reapply such mutations from the write-ahead log
+either when the tablets from the failed server are reassigned by the Master (in the
+case of a single TabletServer failure) or the next time Accumulo starts (in the event of
+failure during shutdown).
+
+Recovery is performed by asking a tablet server to sort the logs so that tablets can easily find their missing
+updates. The sort status of each file is displayed on
+Accumulo monitor status page. Once the recovery is complete any
+tablets involved should return to an ``online'' state. Until then those tablets will be
+unavailable to clients.
+
+The Accumulo client library is configured to retry failed mutations and in many
+cases clients will be able to continue processing after the recovery process without
+throwing an exception.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/analytics.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/analytics.txt b/docs/src/main/asciidoc/chapters/analytics.txt
new file mode 100644
index 0000000..00e0403
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/analytics.txt
@@ -0,0 +1,229 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Analytics
+
+Accumulo supports more advanced data processing than simply keeping keys
+sorted and performing efficient lookups. Analytics can be developed by using
+MapReduce and Iterators in conjunction with Accumulo tables.
+
+=== MapReduce
+
+Accumulo tables can be used as the source and destination of MapReduce jobs. To
+use an Accumulo table with a MapReduce job (specifically with the new Hadoop API
+as of version 0.20), configure the job parameters to use the AccumuloInputFormat
+and AccumuloOutputFormat. Accumulo specific parameters can be set via these
+two format classes to do the following:
+
+* Authenticate and provide user credentials for the input
+* Restrict the scan to a range of rows
+* Restrict the input to a subset of available columns
+
+==== Mapper and Reducer classes
+
+To read from an Accumulo table create a Mapper with the following class
+parameterization and be sure to configure the AccumuloInputFormat.
+
+[source,java]
+class MyMapper extends Mapper<Key,Value,WritableComparable,Writable> {
+ public void map(Key k, Value v, Context c) {
+ // transform key and value data here
+ }
+}
+
+To write to an Accumulo table, create a Reducer with the following class
+parameterization and be sure to configure the AccumuloOutputFormat. The key
+emitted from the Reducer identifies the table to which the mutation is sent. This
+allows a single Reducer to write to more than one table if desired. A default table
+can be configured using the AccumuloOutputFormat, in which case the output table
+name does not have to be passed to the Context object within the Reducer.
+
+[source,java]
+class MyReducer extends Reducer<WritableComparable, Writable, Text, Mutation> {
+ public void reduce(WritableComparable key, Iterable<Text> values, Context c) {
+ Mutation m;
+ // create the mutation based on input key and value
+ c.write(new Text("output-table"), m);
+ }
+}
+
+The Text object passed as the output should contain the name of the table to which
+this mutation should be applied. The Text can be null in which case the mutation
+will be applied to the default table name specified in the AccumuloOutputFormat
+options.
+
+==== AccumuloInputFormat options
+
+[source,java]
+----
+Job job = new Job(getConf());
+AccumuloInputFormat.setInputInfo(job,
+ "user",
+ "passwd".getBytes(),
+ "table",
+ new Authorizations());
+
+AccumuloInputFormat.setZooKeeperInstance(job, "myinstance",
+ "zooserver-one,zooserver-two");
+----
+
+*Optional Settings:*
+
+To restrict Accumulo to a set of row ranges:
+
+[source,java]
+ArrayList<Range> ranges = new ArrayList<Range>();
+// populate array list of row ranges ...
+AccumuloInputFormat.setRanges(job, ranges);
+
+To restrict Accumulo to a list of columns:
+
+[source,java]
+ArrayList<Pair<Text,Text>> columns = new ArrayList<Pair<Text,Text>>();
+// populate list of columns
+AccumuloInputFormat.fetchColumns(job, columns);
+
+To use a regular expression to match row IDs:
+
+[source,java]
+IteratorSetting is = new IteratorSetting(30, RexExFilter.class);
+RegExFilter.setRegexs(is, ".*suffix", null, null, null, true);
+AccumuloInputFormat.addIterator(job, is);
+
+==== AccumuloMultiTableInputFormat options
+
+The AccumuloMultiTableInputFormat allows the scanning over multiple tables
+in a single MapReduce job. Separate ranges, columns, and iterators can be
+used for each table.
+
+[source,java]
+InputTableConfig tableOneConfig = new InputTableConfig();
+InputTableConfig tableTwoConfig = new InputTableConfig();
+
+To set the configuration objects on the job:
+
+[source,java]
+Map<String, InputTableConfig> configs = new HashMap<String,InputTableConfig>();
+configs.put("table1", tableOneConfig);
+configs.put("table2", tableTwoConfig);
+AccumuloMultiTableInputFormat.setInputTableConfigs(job, configs);
+
+*Optional settings:*
+
+To restrict to a set of ranges:
+
+[source,java]
+ArrayList<Range> tableOneRanges = new ArrayList<Range>();
+ArrayList<Range> tableTwoRanges = new ArrayList<Range>();
+// populate array lists of row ranges for tables...
+tableOneConfig.setRanges(tableOneRanges);
+tableTwoConfig.setRanges(tableTwoRanges);
+
+To restrict Accumulo to a list of columns:
+
+[source,java]
+ArrayList<Pair<Text,Text>> tableOneColumns = new ArrayList<Pair<Text,Text>>();
+ArrayList<Pair<Text,Text>> tableTwoColumns = new ArrayList<Pair<Text,Text>>();
+// populate lists of columns for each of the tables ...
+tableOneConfig.fetchColumns(tableOneColumns);
+tableTwoConfig.fetchColumns(tableTwoColumns);
+
+To set scan iterators:
+
+[source,java]
+List<IteratorSetting> tableOneIterators = new ArrayList<IteratorSetting>();
+List<IteratorSetting> tableTwoIterators = new ArrayList<IteratorSetting>();
+// populate the lists of iterator settings for each of the tables ...
+tableOneConfig.setIterators(tableOneIterators);
+tableTwoConfig.setIterators(tableTwoIterators);
+
+
+The name of the table can be retrieved from the input split:
+
+[source,java]
+class MyMapper extends Mapper<Key,Value,WritableComparable,Writable> {
+ public void map(Key k, Value v, Context c) {
+ RangeInputSplit split = (RangeInputSplit)c.getInputSplit();
+ String tableName = split.getTableName();
+ // do something with table name
+ }
+}
+
+
+==== AccumuloOutputFormat options
+
+[source,java]
+----
+boolean createTables = true;
+String defaultTable = "mytable";
+
+AccumuloOutputFormat.setOutputInfo(job,
+ "user",
+ "passwd".getBytes(),
+ createTables,
+ defaultTable);
+
+AccumuloOutputFormat.setZooKeeperInstance(job, "myinstance",
+ "zooserver-one,zooserver-two");
+----
+
+*Optional Settings:*
+
+[source,java]
+AccumuloOutputFormat.setMaxLatency(job, 300000); // milliseconds
+AccumuloOutputFormat.setMaxMutationBufferSize(job, 50000000); // bytes
+
+An example of using MapReduce with Accumulo can be found at
++accumulo/docs/examples/README.mapred+.
+
+=== Combiners
+
+Many applications can benefit from the ability to aggregate values across common
+keys. This can be done via Combiner iterators and is similar to the Reduce step in
+MapReduce. This provides the ability to define online, incrementally updated
+analytics without the overhead or latency associated with batch-oriented
+MapReduce jobs.
+
+All that is needed to aggregate values of a table is to identify the fields over which
+values will be grouped, insert mutations with those fields as the key, and configure
+the table with a combining iterator that supports the summarizing operation
+desired.
+
+The only restriction on an combining iterator is that the combiner developer
+should not assume that all values for a given key have been seen, since new
+mutations can be inserted at anytime. This precludes using the total number of
+values in the aggregation such as when calculating an average, for example.
+
+==== Feature Vectors
+
+An interesting use of combining iterators within an Accumulo table is to store
+feature vectors for use in machine learning algorithms. For example, many
+algorithms such as k-means clustering, support vector machines, anomaly detection,
+etc. use the concept of a feature vector and the calculation of distance metrics to
+learn a particular model. The columns in an Accumulo table can be used to efficiently
+store sparse features and their weights to be incrementally updated via the use of an
+combining iterator.
+
+=== Statistical Modeling
+
+Statistical models that need to be updated by many machines in parallel could be
+similarly stored within an Accumulo table. For example, a MapReduce job that is
+iteratively updating a global statistical model could have each map or reduce worker
+reference the parts of the model to be read and updated through an embedded
+Accumulo client.
+
+Using Accumulo this way enables efficient and fast lookups and updates of small
+pieces of information in a random access pattern, which is complementary to
+MapReduce's sequential access model.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/clients.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/clients.txt b/docs/src/main/asciidoc/chapters/clients.txt
new file mode 100644
index 0000000..3e0b845
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/clients.txt
@@ -0,0 +1,320 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Writing Accumulo Clients
+
+=== Running Client Code
+
+There are multiple ways to run Java code that uses Accumulo. Below is a list
+of the different ways to execute client code.
+
+* using java executable
+* using the accumulo script
+* using the tool script
+
+In order to run client code written to run against Accumulo, you will need to
+include the jars that Accumulo depends on in your classpath. Accumulo client
+code depends on Hadoop and Zookeeper. For Hadoop add the hadoop client jar, all
+of the jars in the Hadoop lib directory, and the conf directory to the
+classpath. For Zookeeper 3.3 you only need to add the Zookeeper jar, and not
+what is in the Zookeeper lib directory. You can run the following command on a
+configured Accumulo system to see what its using for its classpath.
+
+ $ACCUMULO_HOME/bin/accumulo classpath
+
+Another option for running your code is to put a jar file in
++$ACCUMULO_HOME/lib/ext+. After doing this you can use the accumulo
+script to execute your code. For example if you create a jar containing the
+class +com.foo.Client+ and placed that in +lib/ext+, then you could use the command
++$ACCUMULO_HOME/bin/accumulo com.foo.Client+ to execute your code.
+
+If you are writing map reduce job that access Accumulo, then you can use the
+bin/tool.sh script to run those jobs. See the map reduce example.
+
+=== Connecting
+
+All clients must first identify the Accumulo instance to which they will be
+communicating. Code to do this is as follows:
+
+[source,java]
+----
+String instanceName = "myinstance";
+String zooServers = "zooserver-one,zooserver-two"
+Instance inst = new ZooKeeperInstance(instanceName, zooServers);
+
+Connector conn = inst.getConnector("user", new PasswordToken("passwd"));
+----
+
+=== Writing Data
+
+Data are written to Accumulo by creating Mutation objects that represent all the
+changes to the columns of a single row. The changes are made atomically in the
+TabletServer. Clients then add Mutations to a BatchWriter which submits them to
+the appropriate TabletServers.
+
+Mutations can be created thus:
+
+[source,java]
+----
+Text rowID = new Text("row1");
+Text colFam = new Text("myColFam");
+Text colQual = new Text("myColQual");
+ColumnVisibility colVis = new ColumnVisibility("public");
+long timestamp = System.currentTimeMillis();
+
+Value value = new Value("myValue".getBytes());
+
+Mutation mutation = new Mutation(rowID);
+mutation.put(colFam, colQual, colVis, timestamp, value);
+----
+
+==== BatchWriter
+The BatchWriter is highly optimized to send Mutations to multiple TabletServers
+and automatically batches Mutations destined for the same TabletServer to
+amortize network overhead. Care must be taken to avoid changing the contents of
+any Object passed to the BatchWriter since it keeps objects in memory while
+batching.
+
+Mutations are added to a BatchWriter thus:
+
+[source,java]
+----
+// BatchWriterConfig has reasonable defaults
+BatchWriterConfig config = new BatchWriterConfig();
+config.setMaxMemory(10000000L); // bytes available to batchwriter for buffering mutations
+
+BatchWriter writer = conn.createBatchWriter("table", config)
+
+writer.add(mutation);
+
+writer.close();
+----
+
+An example of using the batch writer can be found at
++accumulo/docs/examples/README.batch+.
+
+==== ConditionalWriter
+The ConditionalWriter enables efficient, atomic read-modify-write operations on
+rows. The ConditionalWriter writes special Mutations which have a list of per
+column conditions that must all be met before the mutation is applied. The
+conditions are checked in the tablet server while a row lock is
+held\footnote{Mutations written by the BatchWriter will not obtain a row
+lock.}. The conditions that can be checked for a column are equality and
+absence. For example a conditional mutation can require that column A is
+absent inorder to be applied. Iterators can be applied when checking
+conditions. Using iterators, many other operations besides equality and
+absence can be checked. For example, using an iterator that converts values
+less than 5 to 0 and everything else to 1, its possible to only apply a
+mutation when a column is less than 5.
+
+In the case when a tablet server dies after a client sent a conditional
+mutation, its not known if the mutation was applied or not. When this happens
+the ConditionalWriter reports a status of UNKNOWN for the ConditionalMutation.
+In many cases this situation can be dealt with by simply reading the row again
+and possibly sending another conditional mutation. If this is not sufficient,
+then a higher level of abstraction can be built by storing transactional
+information within a row.
+
+An example of using the batch writer can be found at
++accumulo/docs/examples/README.reservations+.
+
+=== Reading Data
+
+Accumulo is optimized to quickly retrieve the value associated with a given key, and
+to efficiently return ranges of consecutive keys and their associated values.
+
+==== Scanner
+
+To retrieve data, Clients use a Scanner, which acts like an Iterator over
+keys and values. Scanners can be configured to start and stop at particular keys, and
+to return a subset of the columns available.
+
+[source,java]
+----
+// specify which visibilities we are allowed to see
+Authorizations auths = new Authorizations("public");
+
+Scanner scan =
+ conn.createScanner("table", auths);
+
+scan.setRange(new Range("harry","john"));
+scan.fetchFamily("attributes");
+
+for(Entry<Key,Value> entry : scan) {
+ String row = entry.getKey().getRow();
+ Value value = entry.getValue();
+}
+----
+
+==== Isolated Scanner
+
+Accumulo supports the ability to present an isolated view of rows when
+scanning. There are three possible ways that a row could change in Accumulo :
+
+* a mutation applied to a table
+* iterators executed as part of a minor or major compaction
+* bulk import of new files
+
+Isolation guarantees that either all or none of the changes made by these
+operations on a row are seen. Use the IsolatedScanner to obtain an isolated
+view of an Accumulo table. When using the regular scanner it is possible to see
+a non isolated view of a row. For example if a mutation modifies three
+columns, it is possible that you will only see two of those modifications.
+With the isolated scanner either all three of the changes are seen or none.
+
+The IsolatedScanner buffers rows on the client side so a large row will not
+crash a tablet server. By default rows are buffered in memory, but the user
+can easily supply their own buffer if they wish to buffer to disk when rows are
+large.
+
+For an example, look at the following
+
+ examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java
+
+==== BatchScanner
+
+For some types of access, it is more efficient to retrieve several ranges
+simultaneously. This arises when accessing a set of rows that are not consecutive
+whose IDs have been retrieved from a secondary index, for example.
+
+The BatchScanner is configured similarly to the Scanner; it can be configured to
+retrieve a subset of the columns available, but rather than passing a single Range,
+BatchScanners accept a set of Ranges. It is important to note that the keys returned
+by a BatchScanner are not in sorted order since the keys streamed are from multiple
+TabletServers in parallel.
+
+[source,java]
+----
+ArrayList<Range> ranges = new ArrayList<Range>();
+// populate list of ranges ...
+
+BatchScanner bscan =
+ conn.createBatchScanner("table", auths, 10);
+bscan.setRanges(ranges);
+bscan.fetchFamily("attributes");
+
+for(Entry<Key,Value> entry : scan) {
+ System.out.println(entry.getValue());
+}
+----
+
+An example of the BatchScanner can be found at
++accumulo/docs/examples/README.batch+.
+
+=== Proxy
+
+The proxy API allows the interaction with Accumulo with languages other than Java.
+A proxy server is provided in the codebase and a client can further be generated.
+
+==== Prequisites
+
+The proxy server can live on any node in which the basic client API would work. That
+means it must be able to communicate with the Master, ZooKeepers, NameNode, and the
+DataNodes. A proxy client only needs the ability to communicate with the proxy server.
+
+
+==== Configuration
+
+The configuration options for the proxy server live inside of a properties file. At
+the very least, you need to supply the following properties:
+
+ protocolFactory=org.apache.thrift.protocol.TCompactProtocol$Factory
+ tokenClass=org.apache.accumulo.core.client.security.tokens.PasswordToken
+ port=42424
+ instance=test
+ zookeepers=localhost:2181
+
+You can find a sample configuration file in your distribution:
+
+ $ACCUMULO_HOME/proxy/proxy.properties.
+
+This sample configuration file further demonstrates an ability to back the proxy server
+by MockAccumulo or the MiniAccumuloCluster.
+
+==== Running the Proxy Server
+
+After the properties file holding the configuration is created, the proxy server
+can be started using the following command in the Accumulo distribution (assuming
+your properties file is named +config.properties+):
+
+ $ACCUMULO_HOME/bin/accumulo proxy -p config.properties
+
+==== Creating a Proxy Client
+
+Aside from installing the Thrift compiler, you will also need the language-specific library
+for Thrift installed to generate client code in that language. Typically, your operating
+system's package manager will be able to automatically install these for you in an expected
+location such as +/usr/lib/python/site-packages/thrift+.
+
+You can find the thrift file for generating the client:
+
+ $ACCUMULO_HOME/proxy/proxy.thrift.
+
+After a client is generated, the port specified in the configuration properties above will be
+used to connect to the server.
+
+==== Using a Proxy Client
+
+The following examples have been written in Java and the method signatures may be
+slightly different depending on the language specified when generating client with
+the Thrift compiler. After initiating a connection to the Proxy (see Apache Thrift's
+documentation for examples of connecting to a Thrift service), the methods on the
+proxy client will be available. The first thing to do is log in:
+
+[source,java]
+Map password = new HashMap<String,String>();
+password.put("password", "secret");
+ByteBuffer token = client.login("root", password);
+
+Once logged in, the token returned will be used for most subsequent calls to the client.
+Let's create a table, add some data, scan the table, and delete it.
+
+
+First, create a table.
+
+[source,java]
+client.createTable(token, "myTable", true, TimeType.MILLIS);
+
+
+Next, add some data:
+
+[source,java]
+----
+// first, create a writer on the server
+String writer = client.createWriter(token, "myTable", new WriterOptions());
+
+// build column updates
+Map<ByteBuffer, List<ColumnUpdate> cells> cellsToUpdate = //...
+
+// send updates to the server
+client.updateAndFlush(writer, "myTable", cellsToUpdate);
+
+client.closeWriter(writer);
+----
+
+
+Scan for the data and batch the return of the results on the server:
+
+[source,java]
+----
+String scanner = client.createScanner(token, "myTable", new ScanOptions());
+ScanResult results = client.nextK(scanner, 100);
+
+for(KeyValue keyValue : results.getResultsIterator()) {
+ // do something with results
+}
+
+client.closeScanner(scanner);
+----
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/design.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/design.txt b/docs/src/main/asciidoc/chapters/design.txt
new file mode 100644
index 0000000..f57afc3
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/design.txt
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Accumulo Design
+
+=== Data Model
+
+Accumulo provides a richer data model than simple key-value stores, but is not a
+fully relational database. Data is represented as key-value pairs, where the key and
+value are comprised of the following elements:
+
+[width="75%",cols="^,^,^,^,^,^"]
+|===========================================================================
+ 5+|Key .3+^.^|Value
+.2+^.^|Row ID 3+|Column .2+^.^|Timestamp
+ |Family |Qualifier |Visibility
+|===========================================================================
+
+All elements of the Key and the Value are represented as byte arrays except for
+Timestamp, which is a Long. Accumulo sorts keys by element and lexicographically
+in ascending order. Timestamps are sorted in descending order so that later
+versions of the same Key appear first in a sequential scan. Tables consist of a set of
+sorted key-value pairs.
+
+=== Architecture
+
+Accumulo is a distributed data storage and retrieval system and as such consists of
+several architectural components, some of which run on many individual servers.
+Much of the work Accumulo does involves maintaining certain properties of the
+data, such as organization, availability, and integrity, across many commodity-class
+machines.
+
+=== Components
+
+An instance of Accumulo includes many TabletServers, one Garbage Collector process,
+one Master server and many Clients.
+
+==== Tablet Server
+
+The TabletServer manages some subset of all the tablets (partitions of tables). This includes receiving writes from clients, persisting writes to a
+write-ahead log, sorting new key-value pairs in memory, periodically
+flushing sorted key-value pairs to new files in HDFS, and responding
+to reads from clients, forming a merge-sorted view of all keys and
+values from all the files it has created and the sorted in-memory
+store.
+
+TabletServers also perform recovery of a tablet
+that was previously on a server that failed, reapplying any writes
+found in the write-ahead log to the tablet.
+
+==== Garbage Collector
+
+Accumulo processes will share files stored in HDFS. Periodically, the Garbage
+Collector will identify files that are no longer needed by any process, and
+delete them. Multiple garbage collectors can be run to provide hot-standby support.
+They will perform leader election among themselves to choose a single active instance.
+
+==== Master
+
+The Accumulo Master is responsible for detecting and responding to TabletServer
+failure. It tries to balance the load across TabletServer by assigning tablets carefully
+and instructing TabletServers to unload tablets when necessary. The Master ensures all
+tablets are assigned to one TabletServer each, and handles table creation, alteration,
+and deletion requests from clients. The Master also coordinates startup, graceful
+shutdown and recovery of changes in write-ahead logs when Tablet servers fail.
+
+Multiple masters may be run. The masters will choose among themselves a single master,
+and the others will become backups if the master should fail.
+
+==== Tracer
+
+The Accumulo Tracer process supports the distributed timing API provided by Accumulo.
+One to many of these processes can be run on a cluster which will write the timing
+information to a given Accumulo table for future reference. Seeing the section on
+Tracing for more information on this support.
+
+==== Monitor
+
+The Accumulo Monitor is a web application that provides a wealth of information about
+the state of an instance. The Monitor shows graphs and tables which contain information
+about read/write rates, cache hit/miss rates, and Accumulo table information such as scan
+rate and active/queued compactions. Additionally, the Monitor should always be the first
+point of entry when attempting to debug an Accumulo problem as it will show high-level problems
+in addition to aggregated errors from all nodes in the cluster. See the section on Monitoring
+for more information.
+
+Multiple Monitors can be run to provide hot-standby support in the face of failure. Due to the
+forwarding of logs from remote hosts to the Monitor, only one Monitor process should be active
+at one time. Leader election will be performed internally to choose the active Monitor.
+
+==== Client
+
+Accumulo includes a client library that is linked to every application. The client
+library contains logic for finding servers managing a particular tablet, and
+communicating with TabletServers to write and retrieve key-value pairs.
+
+=== Data Management
+
+Accumulo stores data in tables, which are partitioned into tablets. Tablets are
+partitioned on row boundaries so that all of the columns and values for a particular
+row are found together within the same tablet. The Master assigns Tablets to one
+TabletServer at a time. This enables row-level transactions to take place without
+using distributed locking or some other complicated synchronization mechanism. As
+clients insert and query data, and as machines are added and removed from the
+cluster, the Master migrates tablets to ensure they remain available and that the
+ingest and query load is balanced across the cluster.
+
+image::data_distribution.png[width=500]
+
+=== Tablet Service
+
+
+When a write arrives at a TabletServer it is written to a Write-Ahead Log and
+then inserted into a sorted data structure in memory called a MemTable. When the
+MemTable reaches a certain size the TabletServer writes out the sorted key-value
+pairs to a file in HDFS called Indexed Sequential Access Method (ISAM)
+file. This process is called a minor compaction. A new MemTable is then created
+and the fact of the compaction is recorded in the Write-Ahead Log.
+
+When a request to read data arrives at a TabletServer, the TabletServer does a
+binary search across the MemTable as well as the in-memory indexes associated
+with each ISAM file to find the relevant values. If clients are performing a
+scan, several key-value pairs are returned to the client in order from the
+MemTable and the set of ISAM files by performing a merge-sort as they are read.
+
+=== Compactions
+
+In order to manage the number of files per tablet, periodically the TabletServer
+performs Major Compactions of files within a tablet, in which some set of ISAM
+files are combined into one file. The previous files will eventually be removed
+by the Garbage Collector. This also provides an opportunity to permanently
+remove deleted key-value pairs by omitting key-value pairs suppressed by a
+delete entry when the new file is created.
+
+=== Splitting
+
+When a table is created it has one tablet. As the table grows its initial
+tablet eventually splits into two tablets. Its likely that one of these
+tablets will migrate to another tablet server. As the table continues to grow,
+its tablets will continue to split and be migrated. The decision to
+automatically split a tablet is based on the size of a tablets files. The
+size threshold at which a tablet splits is configurable per table. In addition
+to automatic splitting, a user can manually add split points to a table to
+create new tablets. Manually splitting a new table can parallelize reads and
+writes giving better initial performance without waiting for automatic
+splitting.
+
+As data is deleted from a table, tablets may shrink. Over time this can lead
+to small or empty tablets. To deal with this, merging of tablets was
+introduced in Accumulo 1.4. This is discussed in more detail later.
+
+=== Fault-Tolerance
+
+If a TabletServer fails, the Master detects it and automatically reassigns the tablets
+assigned from the failed server to other servers. Any key-value pairs that were in
+memory at the time the TabletServer fails are automatically reapplied from the Write-Ahead
+Log(WAL) to prevent any loss of data.
+
+Tablet servers write their WALs directly to HDFS so the logs are available to all tablet
+servers for recovery. To make the recovery process efficient, the updates within a log are
+grouped by tablet. TabletServers can quickly apply the mutations from the sorted logs
+that are destined for the tablets they have now been assigned.
+
+TabletServer failures are noted on the Master's monitor page, accessible via
++http://master-address:50095/monitor+.
+
+image::failure_handling.png[width=500]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/development_clients.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/development_clients.txt b/docs/src/main/asciidoc/chapters/development_clients.txt
new file mode 100644
index 0000000..fe15c7b
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/development_clients.txt
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Development Clients
+
+Normally, Accumulo consists of lots of moving parts. Even a stand-alone version of
+Accumulo requires Hadoop, Zookeeper, the Accumulo master, a tablet server, etc. If
+you want to write a unit test that uses Accumulo, you need a lot of infrastructure
+in place before your test can run.
+
+=== Mock Accumulo
+
+Mock Accumulo supplies mock implementations for much of the client API. It presently
+does not enforce users, logins, permissions, etc. It does support Iterators and Combiners.
+Note that MockAccumulo holds all data in memory, and will not retain any data or
+settings between runs.
+
+While normal interaction with the Accumulo client looks like this:
+
+[source,java]
+Instance instance = new ZooKeeperInstance(...);
+Connector conn = instance.getConnector(user, passwordToken);
+
+To interact with the MockAccumulo, just replace the ZooKeeperInstance with MockInstance:
+
+[source,java]
+Instance instance = new MockInstance();
+
+In fact, you can use the +--fake+ option to the Accumulo shell and interact with
+MockAccumulo:
+
+----
+$ ./bin/accumulo shell --fake -u root -p ''
+
+Shell - Apache Accumulo Interactive Shell
+-
+- version: 1.6
+- instance name: fake
+- instance id: mock-instance-id
+-
+- type 'help' for a list of available commands
+-
+
+root@fake> createtable test
+
+root@fake test> insert row1 cf cq value
+root@fake test> insert row2 cf cq value2
+root@fake test> insert row3 cf cq value3
+
+root@fake test> scan
+row1 cf:cq [] value
+row2 cf:cq [] value2
+row3 cf:cq [] value3
+
+root@fake test> scan -b row2 -e row2
+row2 cf:cq [] value2
+
+root@fake test>
+----
+
+When testing Map Reduce jobs, you can also set the Mock Accumulo on the AccumuloInputFormat
+and AccumuloOutputFormat classes:
+
+[source,java]
+// ... set up job configuration
+AccumuloInputFormat.setMockInstance(job, "mockInstance");
+AccumuloOutputFormat.setMockInstance(job, "mockInstance");
+
+=== Mini Accumulo Cluster
+
+While the Mock Accumulo provides a lightweight implementation of the client API for unit
+testing, it is often necessary to write more realistic end-to-end integration tests that
+take advantage of the entire ecosystem. The Mini Accumulo Cluster makes this possible by
+configuring and starting Zookeeper, initializing Accumulo, and starting the Master as well
+as some Tablet Servers. It runs against the local filesystem instead of having to start
+up HDFS.
+
+To start it up, you will need to supply an empty directory and a root password as arguments:
+
+[source,java]
+File tempDirectory = // JUnit and Guava supply mechanisms for creating temp directories
+MiniAccumuloCluster accumulo = new MiniAccumuloCluster(tempDirectory, "password");
+accumulo.start();
+
+Once we have our mini cluster running, we will want to interact with the Accumulo client API:
+
+[source,java]
+Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
+Connector conn = instance.getConnector("root", new PasswordToken("password"));
+
+Upon completion of our development code, we will want to shutdown our MiniAccumuloCluster:
+
+[source,java]
+accumulo.stop();
+// delete your temporary folder