You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mj...@apache.org on 2017/06/20 16:24:43 UTC
[13/19] accumulo-website git commit: Jekyll build from master:c9398c5
http://git-wip-us.apache.org/repos/asf/accumulo-website/blob/eab65f94/docs/unreleased/administration/kerberos.html
----------------------------------------------------------------------
diff --git a/docs/unreleased/administration/kerberos.html b/docs/unreleased/administration/kerberos.html
new file mode 100644
index 0000000..540b3b9
--- /dev/null
+++ b/docs/unreleased/administration/kerberos.html
@@ -0,0 +1,1017 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<link href="https://maxcdn.bootstrapcdn.com/bootswatch/3.3.7/paper/bootstrap.min.css" rel="stylesheet" integrity="sha384-awusxf8AUojygHf2+joICySzB780jVvQaVCAt1clU3QsyAitLGul28Qxb2r1e5g+" crossorigin="anonymous">
+<link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.css">
+<link href="/css/accumulo.css" rel="stylesheet" type="text/css">
+
+<title>Accumulo Documentation - Kerberos</title>
+
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script>
+<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
+<script type="text/javascript" src="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.js"></script>
+<script>
+ // show location of canonical site if not currently on the canonical site
+ $(function() {
+ var host = window.location.host;
+ if (typeof host !== 'undefined' && host !== 'accumulo.apache.org') {
+ $('#non-canonical').show();
+ }
+ });
+
+ $(function() {
+ // decorate section headers with anchors
+ return $("h2, h3, h4, h5, h6").each(function(i, el) {
+ var $el, icon, id;
+ $el = $(el);
+ id = $el.attr('id');
+ icon = '<i class="fa fa-link"></i>';
+ if (id) {
+ return $el.append($("<a />").addClass("header-link").attr("href", "#" + id).html(icon));
+ }
+ });
+ });
+
+ // fix sidebar width in documentation
+ $(function() {
+ var $affixElement = $('div[data-spy="affix"]');
+ $affixElement.width($affixElement.parent().width());
+ });
+
+ // configure Google Analytics
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ if (ga.hasOwnProperty('loaded') && ga.loaded === true) {
+ ga('create', 'UA-50934829-1', 'apache.org');
+ ga('send', 'pageview');
+ }
+</script>
+
+</head>
+<body style="padding-top: 100px">
+
+ <nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="/"><img id="nav-logo" alt="Apache Accumulo" class="img-responsive" src="/images/accumulo-logo.png" width="200"
+ /></a>
+ </div>
+ <div class="collapse navbar-collapse" id="navbar-items">
+ <ul class="nav navbar-nav">
+ <li class="nav-link"><a href="/downloads">Download</a></li>
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Releases<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/release/accumulo-1.8.1/">1.8.1 (Latest)</a></li>
+ <li><a href="/release/accumulo-1.7.3/">1.7.3</a></li>
+ <li><a href="/release/accumulo-1.6.6/">1.6.6</a></li>
+ <li><a href="/release/">Archive</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/1.8/accumulo_user_manual.html">User Manual (1.8)</a></li>
+ <li><a href="/1.8/apidocs">Javadocs (1.8)</a></li>
+ <li><a href="/1.8/examples">Examples (1.8)</a></li>
+ <li><a href="/features">Features</a></li>
+ <li><a href="/glossary">Glossary</a></li>
+ <li><a href="/external-docs">External Docs</a></li>
+ <li><a href="/docs-archive/">Archive</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Community<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/get_involved">Get Involved</a></li>
+ <li><a href="/mailing_list">Mailing Lists</a></li>
+ <li><a href="/people">People</a></li>
+ <li><a href="/related-projects">Related Projects</a></li>
+ <li><a href="/contributor/">Contributor Guide</a></li>
+ </ul>
+ </li>
+ </ul>
+ <ul class="nav navbar-nav navbar-right">
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Apache Software Foundation<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="https://www.apache.org">Apache Homepage <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/licenses/LICENSE-2.0">License <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/foundation/sponsorship">Sponsorship <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/security">Security <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/foundation/thanks">Thanks <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/foundation/policies/conduct">Code of Conduct <i class="fa fa-external-link"></i></a></li>
+ </ul>
+ </li>
+ </ul>
+ </div>
+ </div>
+</nav>
+
+ <div class="container">
+ <div class="row">
+ <div class="col-md-12">
+
+ <div id="non-canonical" style="display: none; background-color: #F0E68C; padding-left: 1em;">
+ Visit the official site at: <a href="https://accumulo.apache.org">https://accumulo.apache.org</a>
+ </div>
+ <div id="content">
+
+ <div class="row">
+ <div class="col-md-3">
+ <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true" data-spy="affix">
+ <div class="panel panel-default">
+
+
+
+
+
+
+
+
+
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsegetting-started" aria-expanded="false" aria-controls="collapsegetting-started">
+ Getting started
+ </a>
+ </h4>
+ </div>
+ <div id="collapsegetting-started" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/design">Accumulo Design</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/quick-install">Quick Installation</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/clients">Accumulo Clients</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/shell">Accumulo Shell</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/table_design">Table Design</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/table_configuration">Table Configuration</a></div>
+
+ </div>
+ </div>
+
+
+
+
+
+
+
+
+
+
+
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsedevelopment" aria-expanded="false" aria-controls="collapsedevelopment">
+ Development
+ </a>
+ </h4>
+ </div>
+ <div id="collapsedevelopment" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/iterators">Iterators</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/mapreduce">MapReduce</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/proxy">Proxy</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/development_tools">Development Tools</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/sampling">Sampling</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/summaries">Summary Statistics</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/security">Security</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/high_speed_ingest">High-Speed Ingest</a></div>
+
+ </div>
+ </div>
+
+
+
+
+
+
+
+
+
+
+
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapseadministration" aria-expanded="true" aria-controls="collapseadministration">
+ Administration
+ </a>
+ </h4>
+ </div>
+ <div id="collapseadministration" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/in-depth-install">In-depth Installation</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/configuration-management">Configuration Management</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/configuration-properties">Configuration Properties</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/monitoring-metrics">Monitoring & Metrics</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/tracing">Tracing</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/fate">FATE</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/multivolume">Multi-Volume Installations</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/ssl">SSL</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/kerberos">Kerberos</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/replication">Replication</a></div>
+
+ </div>
+ </div>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsetroubleshooting" aria-expanded="false" aria-controls="collapsetroubleshooting">
+ Troubleshooting
+ </a>
+ </h4>
+ </div>
+ <div id="collapsetroubleshooting" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/basic">Basic Troubleshooting</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/advanced">Advanced Troubleshooting</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/tools">Troubleshooting Tools</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/system-metadata-tables">System Metadata Tables</a></div>
+
+ </div>
+ </div>
+
+
+
+ </div>
+ </div>
+ </div>
+ <div class="col-md-9">
+
+ <p><a href="/docs/unreleased/">Accumulo unreleased docs</a> >> Administration >> Kerberos</p>
+
+
+
+ <div class="alert alert-danger" style="margin-bottom: 0px;" role="alert">This documentation is for a future release of Accumulo! <a href="/1.8/accumulo_user_manual.html">View documentation for the latest release</a>.</div>
+
+ <div class="row">
+ <div class="col-md-10"><h1>Kerberos</h1></div>
+ <div class="col-md-2"><a class="pull-right" style="margin-top: 25px;" href="https://github.com/apache/accumulo-website/edit/master/_docs-unreleased/administration/kerberos.md" role="button"><i class="glyphicon glyphicon-pencil"></i> <small>Edit this page</small></a></div>
+ </div>
+
+ <h2 id="overview">Overview</h2>
+
+<p>Kerberos is a network authentication protocol that provides a secure way for
+peers to prove their identity over an unsecure network in a client-server model.
+A centralized key-distribution center (KDC) is the service that coordinates
+authentication between a client and a server. Clients and servers use “tickets”,
+obtained from the KDC via a password or a special file called a “keytab”, to
+communicate with the KDC and prove their identity. A KDC administrator must
+create the principal (name for the client/server identiy) and the password
+or keytab, securely passing the necessary information to the actual user/service.
+Properly securing the KDC and generated ticket material is central to the security
+model and is mentioned only as a warning to administrators running their own KDC.</p>
+
+<p>To interact with Kerberos programmatically, GSSAPI and SASL are two standards
+which allow cross-language integration with Kerberos for authentication. GSSAPI,
+the generic security service application program interface, is a standard which
+Kerberos implements. In the Java programming language, the language itself also implements
+GSSAPI which is leveraged by other applications, like Apache Hadoop and Apache Thrift.
+SASL, simple authentication and security layer, is a framework for authentication and
+and security over the network. SASL provides a number of mechanisms for authentication,
+one of which is GSSAPI. Thus, SASL provides the transport which authenticates
+using GSSAPI that Kerberos implements.</p>
+
+<p>Kerberos is a very complicated software application and is deserving of much
+more description than can be provided here. An <a href="http://www.roguelynn.com/words/explain-like-im-5-kerberos/">explain like I`m 5</a>
+blog post is very good at distilling the basics, while <a href="http://web.mit.edu/kerberos/">MIT Keberos’s project page</a>
+contains lots of documentation for users or administrators. Various Hadoop “vendors”
+also provide free documentation that includes step-by-step instructions for
+configuring Hadoop and ZooKeeper (which will be henceforth considered as prerequisites).</p>
+
+<h2 id="within-hadoop">Within Hadoop</h2>
+
+<p>Out of the box, HDFS and YARN have no ability to enforce that a user is who
+they claim they are. Thus, any basic Hadoop installation should be treated as
+unsecure: any user with access to the cluster has the ability to access any data.
+Using Kerberos to provide authentication, users can be strongly identified, delegating
+to Kerberos to determine who a user is and enforce that a user is who they claim to be.
+As such, Kerberos is widely used across the entire Hadoop ecosystem for strong
+authentication. Since server processes accessing HDFS or YARN are required
+to use Kerberos to authenticate with HDFS, it makes sense that they also require
+Kerberos authentication from their clients, in addition to other features provided
+by SASL.</p>
+
+<p>A typical deployment involves the creation of Kerberos principals for all server
+processes (Hadoop datanodes and namenode(s), ZooKeepers), the creation of a keytab
+file for each principal and then proper configuration for the Hadoop site xml files.
+Users also need Kerberos principals created for them; however, a user typically
+uses a password to identify themselves instead of a keytab. Users can obtain a
+ticket granting ticket (TGT) from the KDC using their password which allows them
+to authenticate for the lifetime of the TGT (typically one day by default) and alleviates
+the need for further password authentication.</p>
+
+<p>For client server applications, like web servers, a keytab can be created which
+allow for fully-automated Kerberos identification removing the need to enter any
+password, at the cost of needing to protect the keytab file. These principals
+will apply directly to authentication for clients accessing Accumulo and the
+Accumulo processes accessing HDFS.</p>
+
+<h2 id="delegation-tokens">Delegation Tokens</h2>
+
+<p>MapReduce, a common way that clients interact with Accumulo, does not map well to the
+client-server model that Kerberos was originally designed to support. Specifically, the parallelization
+of tasks across many nodes introduces the problem of securely sharing the user credentials across
+these tasks in as safe a manner as possible. To address this problem, Hadoop introduced the notion
+of a delegation token to be used in distributed execution settings.</p>
+
+<p>A delegation token is nothing more than a short-term, on-the-fly password generated after authenticating with the user’s
+credentials. In Hadoop itself, the Namenode and ResourceManager, for HDFS and YARN respectively, act as the gateway for
+delegation tokens requests. For example, before a YARN job is submitted, the implementation will request delegation
+tokens from the NameNode and ResourceManager so the YARN tasks can communicate with HDFS and YARN. In the same manner,
+support has been added in the Accumulo Master to generate delegation tokens to enable interaction with Accumulo via
+MapReduce when Kerberos authentication is enabled in a manner similar to HDFS and YARN.</p>
+
+<p>Generating an expiring password is, arguably, more secure than distributing the user’s
+credentials across the cluster as only access to HDFS, YARN and Accumulo would be
+compromised in the case of the token being compromised as opposed to the entire
+Kerberos credential. Additional details for clients and servers will be covered
+in subsequent sections.</p>
+
+<h2 id="configuring-accumulo">Configuring Accumulo</h2>
+
+<p>To configure Accumulo for use with Kerberos, both client-facing and server-facing
+changes must be made for a functional system on secured Hadoop. As previously mentioned,
+numerous guidelines already exist on the subject of configuring Hadoop and ZooKeeper for
+use with Kerberos and won’t be covered here. It is assumed that you have functional
+Hadoop and ZooKeeper already installed.</p>
+
+<p>Note that on an existing cluster the server side changes will require a full cluster shutdown and restart. You should
+wait to restart the TraceServers until after you’ve completed the rest of the cluster set up and provisioned
+a trace user with appropriate permissions.</p>
+
+<h3 id="servers">Servers</h3>
+
+<p>The first step is to obtain a Kerberos identity for the Accumulo server processes.
+When running Accumulo with Kerberos enabled, a valid Kerberos identity will be required
+to initiate any RPC between Accumulo processes (e.g. Master and TabletServer) in addition
+to any HDFS action (e.g. client to HDFS or TabletServer to HDFS).</p>
+
+<h4 id="generate-principal-and-keytab">Generate Principal and Keytab</h4>
+
+<p>In the <code class="highlighter-rouge">kadmin.local</code> shell or using the <code class="highlighter-rouge">-q</code> option on <code class="highlighter-rouge">kadmin.local</code>, create a
+principal for Accumulo for all hosts that are running Accumulo processes. A Kerberos
+principal is of the form “primary/instance@REALM”. “accumulo” is commonly the “primary”
+(although not required) and the “instance” is the fully-qualified domain name for
+the host that will be running the Accumulo process – this is required.</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>kadmin.local -q "addprinc -randkey accumulo/host.domain.com"
+</code></pre>
+</div>
+
+<p>Perform the above for each node running Accumulo processes in the instance, modifying
+“host.domain.com” for your network. The <code class="highlighter-rouge">randkey</code> option generates a random password
+because we will use a keytab for authentication, not a password, since the Accumulo
+server processes don’t have an interactive console to enter a password into.</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>kadmin.local -q "xst -k accumulo.hostname.keytab accumulo/host.domain.com"
+</code></pre>
+</div>
+
+<p>To simplify deployments, at thet cost of security, all Accumulo principals could
+be globbed into a single keytab</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>kadmin.local -q "xst -k accumulo.service.keytab -glob accumulo*"
+</code></pre>
+</div>
+
+<p>To ensure that the SASL handshake can occur from clients to servers and servers to servers,
+all Accumulo servers must share the same instance and realm principal components as the
+“client” needs to know these to set up the connection with the “server”.</p>
+
+<h4 id="server-configuration">Server Configuration</h4>
+
+<p>A number of properties need to be changed to account to properly configure servers
+in <code class="highlighter-rouge">accumulo-site.xml</code>.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Key</th>
+ <th>Default Value</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>general.kerberos.keytab</td>
+ <td>/etc/security/keytabs/accumulo.service.keytab</td>
+ <td>The path to the keytab for Accumulo on local filesystem. Change the value to the actual path on your system.</td>
+ </tr>
+ <tr>
+ <td>general.kerberos.principal</td>
+ <td>accumulo/_HOST@REALM</td>
+ <td>The Kerberos principal for Accumulo, needs to match the keytab. “_HOST” can be used instead of the actual hostname in the principal and will be automatically expanded to the current FQDN which reduces the configuration file burden.</td>
+ </tr>
+ <tr>
+ <td>instance.rpc.sasl.enabled</td>
+ <td>true</td>
+ <td>Enables SASL for the Thrift Servers (supports GSSAPI)</td>
+ </tr>
+ <tr>
+ <td>rpc.sasl.qop</td>
+ <td>auth</td>
+ <td>One of “auth”, “auth-int”, or “auth-conf”. These map to the SASL defined properties for quality of protection. “auth” is authentication only. “auth-int” is authentication and data integrity. “auth-conf” is authentication, data integrity and confidentiality.</td>
+ </tr>
+ <tr>
+ <td>instance.security.authenticator</td>
+ <td>org.apache.accumulo.server.security.handler.KerberosAuthenticator</td>
+ <td>Configures Accumulo to use the Kerberos principal as the Accumulo username/principal</td>
+ </tr>
+ <tr>
+ <td>instance.security.authorizor</td>
+ <td>org.apache.accumulo.server.security.handler.KerberosAuthorizor</td>
+ <td>Configures Accumulo to use the Kerberos principal for authorization purposes</td>
+ </tr>
+ <tr>
+ <td>instance.security.permissionHandler</td>
+ <td>org.apache.accumulo.server.security.handler.KerberosPermissionHandler</td>
+ <td>Configures Accumulo to use the Kerberos principal for permission purposes</td>
+ </tr>
+ <tr>
+ <td>trace.token.type</td>
+ <td>org.apache.accumulo.core.client.security.tokens.KerberosToken</td>
+ <td>Configures the Accumulo Tracer to use the KerberosToken for authentication when serializing traces to the trace table.</td>
+ </tr>
+ <tr>
+ <td>trace.user</td>
+ <td>accumulo/_HOST@REALM</td>
+ <td>The tracer process needs valid credentials to serialize traces to Accumulo. While the other server processes are creating a SystemToken from the provided keytab and principal, we can still use a normal KerberosToken and the same keytab/principal to serialize traces. Like non-Kerberized instances, the table must be created and permissions granted to the trace.user. The same <code class="highlighter-rouge">_HOST</code> replacement is performed on this value, substituted the FQDN for <code class="highlighter-rouge">_HOST</code>.</td>
+ </tr>
+ <tr>
+ <td>trace.token.property.keytab</td>
+ <td> </td>
+ <td>You can optionally specify the path to a keytab file for the principal given in the <code class="highlighter-rouge">trace.user</code> property. If you don’t set this path, it will default to the value given in <code class="highlighter-rouge">general.kerberos.principal</code>.</td>
+ </tr>
+ <tr>
+ <td>general.delegation.token.lifetime</td>
+ <td>7d</td>
+ <td>The length of time that the server-side secret used to create delegation tokens is valid. After a server-side secret expires, a delegation token created with that secret is no longer valid.</td>
+ </tr>
+ <tr>
+ <td>general.delegation.token.update.interval</td>
+ <td>1d</td>
+ <td>The frequency in which new server-side secrets should be generated to create delegation tokens for clients. Generating new secrets reduces the likelihood of cryptographic attacks.</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>Although it should be a prerequisite, it is ever important that you have DNS properly
+configured for your nodes and that Accumulo is configured to use the FQDN. It
+is extremely important to use the FQDN in each of the “hosts” files for each
+Accumulo process: <code class="highlighter-rouge">masters</code>, <code class="highlighter-rouge">monitors</code>, <code class="highlighter-rouge">tservers</code>, <code class="highlighter-rouge">tracers</code>, and <code class="highlighter-rouge">gc</code>.</p>
+
+<p>Normally, no changes are needed in <code class="highlighter-rouge">accumulo-env.sh</code> to enable Kerberos. Typically, the <code class="highlighter-rouge">krb5.conf</code>
+is installed on the local machine in <code class="highlighter-rouge">/etc/</code>, and the Java library implementations will look
+here to find the necessary configuration to communicate with the KDC. Some installations
+may require a different <code class="highlighter-rouge">krb5.conf</code> to be used for Accumulo which can be accomplished
+by adding the JVM system property <code class="highlighter-rouge">-Djava.security.krb5.conf=/path/to/other/krb5.conf</code> to
+<code class="highlighter-rouge">JAVA_OPTS</code> in <code class="highlighter-rouge">accumulo-env.sh</code>.</p>
+
+<h4 id="kerberosauthenticator">KerberosAuthenticator</h4>
+
+<p>The <code class="highlighter-rouge">KerberosAuthenticator</code> is an implementation of the pluggable security interfaces
+that Accumulo provides. It builds on top of what the default ZooKeeper-based implementation,
+but removes the need to create user accounts with passwords in Accumulo for clients. As
+long as a client has a valid Kerberos identity, they can connect to and interact with
+Accumulo, but without any permissions (e.g. cannot create tables or write data). Leveraging
+ZooKeeper removes the need to change the permission handler and authorizor, so other Accumulo
+functions regarding permissions and cell-level authorizations do not change.</p>
+
+<p>It is extremely important to note that, while user operations like <code class="highlighter-rouge">SecurityOperations.listLocalUsers()</code>,
+<code class="highlighter-rouge">SecurityOperations.dropLocalUser()</code>, and <code class="highlighter-rouge">SecurityOperations.createLocalUser()</code> will not return
+errors, these methods are not equivalent to normal installations, as they will only operate on
+users which have, at one point in time, authenticated with Accumulo using their Kerberos identity.
+The KDC is still the authoritative entity for user management. The previously mentioned methods
+are provided as they simplify management of users within Accumulo, especially with respect
+to granting Authorizations and Permissions to new users.</p>
+
+<h4 id="administrative-user">Administrative User</h4>
+
+<p>Out of the box (without Kerberos enabled), Accumulo has a single user with administrative permissions “root”.
+This users is used to “bootstrap” other users, creating less-privileged users for applications using
+the system. In Kerberos, to authenticate with the system, it’s required that the client presents Kerberos
+credentials for the principal (user) the client is trying to authenticate as.</p>
+
+<p>Because of this, an administrative user named “root” would be useless in an instance using Kerberos,
+because it is very unlikely to have Kerberos credentials for a principal named <code class="highlighter-rouge">root</code>. When Kerberos is
+enabled, Accumulo will prompt for the name of a user to grant the same permissions as what the <code class="highlighter-rouge">root</code>
+user would normally have. The name of the Accumulo user to grant administrative permissions to can
+also be given by the <code class="highlighter-rouge">-u</code> or <code class="highlighter-rouge">--user</code> options.</p>
+
+<p>If you are enabling Kerberos on an existing cluster, you will need to reinitialize the security system in
+order to replace the existing “root” user with one that can be used with Kerberos. These steps should be
+completed after you have done the previously described configuration changes and will require access to
+a complete <code class="highlighter-rouge">accumulo-site.xml</code>, including the instance secret. Note that this process will delete all
+existing users in the system; you will need to reassign user permissions based on Kerberos principals.</p>
+
+<ol>
+ <li>Ensure Accumulo is not running.</li>
+ <li>Given the path to a <code class="highlighter-rouge">accumulo-site.xml</code> with the instance secret, run the security reset tool. If you are
+prompted for a password you can just hit return, since it won’t be used.</li>
+ <li>Start the Accumulo cluster</li>
+</ol>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>$ accumulo-cluster stop
+...
+$ accumulo init --reset-security
+Running against secured HDFS
+Principal (user) to grant administrative privileges to : acculumo_admin@EXAMPLE.COM
+Enter initial password for accumulo_admin@EXAMPLE.COM (this may not be applicable for your security setup):
+Confirm initial password for accumulo_admin@EXAMPLE.COM:
+$ accumulo-cluster start
+...
+</code></pre>
+</div>
+
+<h4 id="verifying-secure-access">Verifying secure access</h4>
+
+<p>To verify that servers have correctly started with Kerberos enabled, ensure that the processes
+are actually running (they should exit immediately if login fails) and verify that you see
+something similar to the following in the application log.</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>2015-01-07 11:57:56,826 [security.SecurityUtil] INFO : Attempting to login with keytab as accumulo/hostname@EXAMPLE.COM
+2015-01-07 11:57:56,830 [security.UserGroupInformation] INFO : Login successful for user accumulo/hostname@EXAMPLE.COM using keytab file /etc/security/keytabs/accumulo.service.keytab
+</code></pre>
+</div>
+
+<h4 id="impersonation">Impersonation</h4>
+
+<p>Impersonation is functionality which allows a certain user to act as another. One direct application
+of this concept within Accumulo is the Thrift proxy. The Thrift proxy is configured to accept
+user requests and pass them onto Accumulo, enabling client access to Accumulo via any thrift-compatible
+language. When the proxy is running with SASL transports, this enforces that clients present a valid
+Kerberos identity to make a connection. In this situation, the Thrift proxy server does not have
+access to the secret key material in order to make a secure connection to Accumulo as the client,
+it can only connect to Accumulo as itself. Impersonation, in this context, refers to the ability
+of the proxy to authenticate to Accumulo as itself, but act on behalf of an Accumulo user.</p>
+
+<p>Accumulo supports basic impersonation of end-users by a third party via static rules in Accumulo’s
+site configuration file. These two properties are semi-colon separated properties which are aligned
+by index. This first element in the user impersonation property value matches the first element
+in the host impersonation property value, etc.</p>
+
+<div class="language-xml highlighter-rouge"><pre class="highlight"><code><span class="nt"><property></span>
+ <span class="nt"><name></span>instance.rpc.sasl.allowed.user.impersonation<span class="nt"></name></span>
+ <span class="nt"><value></span>$PROXY_USER:*<span class="nt"></value></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>instance.rpc.sasl.allowed.host.impersonation<span class="nt"></name></span>
+ <span class="nt"><value></span>*<span class="nt"></value></span>
+<span class="nt"></property></span>
+</code></pre>
+</div>
+
+<p>Here, <code class="highlighter-rouge">$PROXY_USER</code> can impersonate any user from any host.</p>
+
+<p>The following is an example of specifying a subset of users <code class="highlighter-rouge">$PROXY_USER</code> can impersonate and also
+limiting the hosts from which <code class="highlighter-rouge">$PROXY_USER</code> can initiate requests from.</p>
+
+<div class="language-xml highlighter-rouge"><pre class="highlight"><code><span class="nt"><property></span>
+ <span class="nt"><name></span>instance.rpc.sasl.allowed.user.impersonation<span class="nt"></name></span>
+ <span class="nt"><value></span>$PROXY_USER:user1,user2;$PROXY_USER2:user2,user4<span class="nt"></value></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>instance.rpc.sasl.allowed.host.impersonation<span class="nt"></name></span>
+ <span class="nt"><value></span>host1.domain.com,host2.domain.com;*<span class="nt"></value></span>
+<span class="nt"></property></span>
+</code></pre>
+</div>
+
+<p>Here, <code class="highlighter-rouge">$PROXY_USER</code> can impersonate user1 and user2 only from host1.domain.com or host2.domain.com.
+<code class="highlighter-rouge">$PROXY_USER2</code> can impersonate user2 and user4 from any host.</p>
+
+<p>In these examples, the value <code class="highlighter-rouge">$PROXY_USER</code> is the Kerberos principal of the server which is acting on behalf of a user.
+Impersonation is enforced by the Kerberos principal and the host from which the RPC originated (from the perspective
+of the Accumulo TabletServers/Masters). An asterisk (*) can be used to specify all users or all hosts (depending on the context).</p>
+
+<h4 id="delegation-tokens-1">Delegation Tokens</h4>
+
+<p>Within Accumulo services, the primary task to implement delegation tokens is the generation and distribution
+of a shared secret among all Accumulo tabletservers and the master. The secret key allows for generation
+of delegation tokens for users and verification of delegation tokens presented by clients. If a server
+process is unaware of the secret key used to create a delegation token, the client cannot be authenticated.
+As ZooKeeper distribution is an asynchronous operation (typically on the order of seconds), the
+value for <code class="highlighter-rouge">general.delegation.token.update.interval</code> should be on the order of hours to days to reduce the
+likelihood of servers rejecting valid clients because the server did not yet see a new secret key.</p>
+
+<p>Supporting authentication with both Kerberos credentials and delegation tokens, the SASL thrift
+server accepts connections with either <code class="highlighter-rouge">GSSAPI</code> and <code class="highlighter-rouge">DIGEST-MD5</code> mechanisms set. The <code class="highlighter-rouge">DIGEST-MD5</code> mechanism
+enables authentication as a normal username and password exchange which <code class="highlighter-rouge">DelegationToken</code>s leverages.</p>
+
+<p>Since delegation tokens are a weaker form of authentication than Kerberos credentials, user access
+to obtain delegation tokens from Accumulo is protected with the <code class="highlighter-rouge">DELEGATION_TOKEN</code> system permission. Only
+users with the system permission are allowed to obtain delegation tokens. It is also recommended
+to configure confidentiality with SASL, using the <code class="highlighter-rouge">rpc.sasl.qop=auth-conf</code> configuration property, to
+ensure that prying eyes cannot view the <code class="highlighter-rouge">DelegationToken</code> as it passes over the network.</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code># Check a user's permissions
+admin@REALM@accumulo> userpermissions -u user@REALM
+
+# Grant the DELEGATION_TOKEN system permission to a user
+admin@REALM@accumulo> grant System.DELEGATION_TOKEN -s -u user@REALM
+</code></pre>
+</div>
+
+<h3 id="clients">Clients</h3>
+
+<h4 id="create-client-principal">Create client principal</h4>
+
+<p>Like the Accumulo servers, clients must also have a Kerberos principal created for them. The
+primary difference between a server principal is that principals for users are created
+with a password and also not qualified to a specific instance (host).</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>kadmin.local -q "addprinc $user"
+</code></pre>
+</div>
+
+<p>The above will prompt for a password for that user which will be used to identify that $user.
+The user can verify that they can authenticate with the KDC using the command <code class="highlighter-rouge">kinit $user</code>.
+Upon entering the correct password, a local credentials cache will be made which can be used
+to authenticate with Accumulo, access HDFS, etc.</p>
+
+<p>The user can verify the state of their local credentials cache by using the command <code class="highlighter-rouge">klist</code>.</p>
+
+<div class="language-shell highlighter-rouge"><pre class="highlight"><code><span class="gp">$ </span>klist
+Ticket cache: FILE:/tmp/krb5cc_123
+Default principal: user@EXAMPLE.COM
+
+Valid starting Expires Service principal
+01/07/2015 11:56:35 01/08/2015 11:56:35 krbtgt/EXAMPLE.COM@EXAMPLE.COM
+ renew <span class="k">until </span>01/14/2015 11:56:35
+</code></pre>
+</div>
+
+<h4 id="configuration">Configuration</h4>
+
+<p>The second thing clients need to do is to set up their client configuration file. By
+default, this file is stored in <code class="highlighter-rouge">~/.accumulo/config</code> or <code class="highlighter-rouge">/path/to/accumulo/client.conf</code>.
+Accumulo utilities also allow you to provide your own copy of this file in any location
+using the <code class="highlighter-rouge">--config-file</code> command line option.</p>
+
+<p>Three items need to be set to enable access to Accumulo:</p>
+
+<ul>
+ <li><code class="highlighter-rouge">instance.rpc.sasl.enabled</code>=<em>true</em></li>
+ <li><code class="highlighter-rouge">rpc.sasl.qop</code>=<em>auth</em></li>
+ <li><code class="highlighter-rouge">kerberos.server.primary</code>=<em>accumulo</em></li>
+</ul>
+
+<p>Each of these properties <em>must</em> match the configuration of the accumulo servers; this is
+required to set up the SASL transport.</p>
+
+<h4 id="verifying-administrative-access">Verifying Administrative Access</h4>
+
+<p>At this point you should have enough configured on the server and client side to interact with
+the system. You should verify that the administrative user you chose earlier can successfully
+interact with the sytem.</p>
+
+<p>While this example logs in via <code class="highlighter-rouge">kinit</code> with a password, any login method that caches Kerberos tickets
+should work.</p>
+
+<div class="language-shell highlighter-rouge"><pre class="highlight"><code><span class="gp">$ </span>kinit accumulo_admin@EXAMPLE.COM
+Password <span class="k">for </span>accumulo_admin@EXAMPLE.COM: <span class="k">******************************</span>
+<span class="gp">$ </span>accumulo shell
+
+Shell - Apache Accumulo Interactive Shell
+-
+- version: 1.7.2
+- instance name: MYACCUMULO
+- instance id: 483b9038-889f-4b2d-b72b-dfa2bb5dbd07
+-
+- <span class="nb">type</span> <span class="s1">'help'</span> <span class="k">for </span>a list of available commands
+-
+<span class="gp">accumulo_admin@EXAMPLE.COM@MYACCUMULO> </span>userpermissions
+System permissions: System.GRANT, System.CREATE_TABLE, System.DROP_TABLE, System.ALTER_TABLE, System.CREATE_USER, System.DROP_USER, System.ALTER_USER, System.SYSTEM, System.CREATE_NAMESPACE, System.DROP_NAMESPACE, System.ALTER_NAMESPACE, System.OBTAIN_DELEGATION_TOKEN
+
+Namespace permissions <span class="o">(</span>accumulo<span class="o">)</span>: Namespace.READ, Namespace.ALTER_TABLE
+
+Table permissions <span class="o">(</span>accumulo.metadata<span class="o">)</span>: Table.READ, Table.ALTER_TABLE
+Table permissions <span class="o">(</span>accumulo.replication<span class="o">)</span>: Table.READ
+Table permissions <span class="o">(</span>accumulo.root<span class="o">)</span>: Table.READ, Table.ALTER_TABLE
+
+<span class="gp">accumulo_admin@EXAMPLE.COM@MYACCUMULO> </span>quit
+<span class="gp">$ </span>kdestroy
+<span class="err">$</span>
+</code></pre>
+</div>
+
+<h4 id="delegationtokens-with-mapreduce">DelegationTokens with MapReduce</h4>
+
+<p>To use DelegationTokens in a custom MapReduce job, the call to <code class="highlighter-rouge">setConnectorInfo()</code> method
+on <code class="highlighter-rouge">AccumuloInputFormat</code> or <code class="highlighter-rouge">AccumuloOutputFormat</code> should be the only necessary change. Instead
+of providing an instance of a <code class="highlighter-rouge">KerberosToken</code>, the user must call <code class="highlighter-rouge">SecurityOperations.getDelegationToken</code>
+using a <code class="highlighter-rouge">Connector</code> obtained with that <code class="highlighter-rouge">KerberosToken</code>, and pass the <code class="highlighter-rouge">DelegationToken</code> to
+<code class="highlighter-rouge">setConnectorInfo</code> instead of the <code class="highlighter-rouge">KerberosToken</code>. It is expected that the user launching
+the MapReduce job is already logged in via Kerberos via a keytab or via a locally-cached
+Kerberos ticket-granting-ticket (TGT).</p>
+
+<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">Instance</span> <span class="n">instance</span> <span class="o">=</span> <span class="n">getInstance</span><span class="o">();</span>
+<span class="n">KerberosToken</span> <span class="n">kt</span> <span class="o">=</span> <span class="k">new</span> <span class="n">KerberosToken</span><span class="o">();</span>
+<span class="n">Connector</span> <span class="n">conn</span> <span class="o">=</span> <span class="n">instance</span><span class="o">.</span><span class="na">getConnector</span><span class="o">(</span><span class="n">principal</span><span class="o">,</span> <span class="n">kt</span><span class="o">);</span>
+<span class="n">DelegationToken</span> <span class="n">dt</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="na">securityOperations</span><span class="o">().</span><span class="na">getDelegationToken</span><span class="o">();</span>
+
+<span class="c1">// Reading from Accumulo</span>
+<span class="n">AccumuloInputFormat</span><span class="o">.</span><span class="na">setConnectorInfo</span><span class="o">(</span><span class="n">job</span><span class="o">,</span> <span class="n">principal</span><span class="o">,</span> <span class="n">dt</span><span class="o">);</span>
+
+<span class="c1">// Writing to Accumulo</span>
+<span class="n">AccumuloOutputFormat</span><span class="o">.</span><span class="na">setConnectorInfo</span><span class="o">(</span><span class="n">job</span><span class="o">,</span> <span class="n">principal</span><span class="o">,</span> <span class="n">dt</span><span class="o">);</span>
+</code></pre>
+</div>
+
+<p>If the user passes a <code class="highlighter-rouge">KerberosToken</code> to the <code class="highlighter-rouge">setConnectorInfo</code> method, the implementation will
+attempt to obtain a <code class="highlighter-rouge">DelegationToken</code> automatically, but this does have limitations
+based on the other MapReduce configuration methods already called and permissions granted
+to the calling user. It is best for the user to acquire the DelegationToken on their own
+and provide it directly to <code class="highlighter-rouge">setConnectorInfo</code>.</p>
+
+<p>Users must have the <code class="highlighter-rouge">DELEGATION_TOKEN</code> system permission to call the <code class="highlighter-rouge">getDelegationToken</code>
+method. The obtained delegation token is only valid for the requesting user for a period
+of time dependent on Accumulo’s configuration (<code class="highlighter-rouge">general.delegation.token.lifetime</code>).</p>
+
+<p>It is also possible to obtain and use <code class="highlighter-rouge">DelegationToken</code>s outside of the context
+of MapReduce.</p>
+
+<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">String</span> <span class="n">principal</span> <span class="o">=</span> <span class="s">"user@REALM"</span><span class="o">;</span>
+<span class="n">Instance</span> <span class="n">instance</span> <span class="o">=</span> <span class="n">getInstance</span><span class="o">();</span>
+<span class="n">Connector</span> <span class="n">connector</span> <span class="o">=</span> <span class="n">instance</span><span class="o">.</span><span class="na">getConnector</span><span class="o">(</span><span class="n">principal</span><span class="o">,</span> <span class="k">new</span> <span class="n">KerberosToken</span><span class="o">());</span>
+<span class="n">DelegationToken</span> <span class="n">delegationToken</span> <span class="o">=</span> <span class="n">connector</span><span class="o">.</span><span class="na">securityOperations</span><span class="o">().</span><span class="na">getDelegationToken</span><span class="o">();</span>
+
+<span class="n">Connector</span> <span class="n">dtConnector</span> <span class="o">=</span> <span class="n">instance</span><span class="o">.</span><span class="na">getConnector</span><span class="o">(</span><span class="n">principal</span><span class="o">,</span> <span class="n">delegationToken</span><span class="o">);</span>
+</code></pre>
+</div>
+
+<p>Use of the <code class="highlighter-rouge">dtConnector</code> will perform each operation as the original user, but without
+their Kerberos credentials.</p>
+
+<p>For the duration of validity of the <code class="highlighter-rouge">DelegationToken</code>, the user <em>must</em> take the necessary precautions
+to protect the <code class="highlighter-rouge">DelegationToken</code> from prying eyes as it can be used by any user on any host to impersonate
+the user who requested the <code class="highlighter-rouge">DelegationToken</code>. YARN ensures that passing the delegation token from the client
+JVM to each YARN task is secure, even in multi-tenant instances.</p>
+
+<h3 id="debugging">Debugging</h3>
+
+<p><strong>Q</strong>: I have valid Kerberos credentials and a correct client configuration file but
+I still get errors like:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>java.io.IOException: Failed on local exception: java.io.IOException: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
+</code></pre>
+</div>
+
+<p><strong>A</strong>: When you have a valid client configuration and Kerberos TGT, it is possible that the search
+path for your local credentials cache is incorrect. Check the value of the KRB5CCNAME environment
+value, and ensure it matches the value reported by <code class="highlighter-rouge">klist</code>.</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>$ echo $KRB5CCNAME
+
+$ klist
+Ticket cache: FILE:/tmp/krb5cc_123
+Default principal: user@EXAMPLE.COM
+
+Valid starting Expires Service principal
+01/07/2015 11:56:35 01/08/2015 11:56:35 krbtgt/EXAMPLE.COM@EXAMPLE.COM
+ renew until 01/14/2015 11:56:35
+$ export KRB5CCNAME=/tmp/krb5cc_123
+$ echo $KRB5CCNAME
+/tmp/krb5cc_123
+</code></pre>
+</div>
+
+<p><strong>Q</strong>: I thought I had everything configured correctly, but my client/server still fails to log in.
+I don’t know what is actually failing.</p>
+
+<p><strong>A</strong>: Add the following system property to the JVM invocation:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>-Dsun.security.krb5.debug=true
+</code></pre>
+</div>
+
+<p>This will enable lots of extra debugging at the JVM level which is often sufficient to
+diagnose some high-level configuration problem. Client applications can add this system property by
+hand to the command line and Accumulo server processes or applications started using the <code class="highlighter-rouge">accumulo</code>
+script by adding the property to <code class="highlighter-rouge">JAVA_OPTS</code> in <code class="highlighter-rouge">accumulo-env.sh</code>.</p>
+
+<p>Additionally, you can increase the log4j levels on <code class="highlighter-rouge">org.apache.hadoop.security</code>, which includes the
+Hadoop <code class="highlighter-rouge">UserGroupInformation</code> class, which will include some high-level debug statements. This
+can be controlled in your client application, or using <code class="highlighter-rouge">log4j-service.properties</code></p>
+
+<p><strong>Q</strong>: All of my Accumulo processes successfully start and log in with their
+keytab, but they are unable to communicate with each other, showing the
+following errors:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>2015-01-12 14:47:27,055 [transport.TSaslTransport] ERROR: SASL negotiation failure
+javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Server not found in Kerberos database (7) - LOOKING_UP_SERVER)]
+ at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:212)
+ at org.apache.thrift.transport.TSaslClientTransport.handleSaslStartMessage(TSaslClientTransport.java:94)
+ at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:253)
+ at org.apache.thrift.transport.TSaslClientTransport.open(TSaslClientTransport.java:37)
+ at org.apache.accumulo.core.rpc.UGIAssumingTransport$1.run(UGIAssumingTransport.java:53)
+ at org.apache.accumulo.core.rpc.UGIAssumingTransport$1.run(UGIAssumingTransport.java:49)
+ at java.security.AccessController.doPrivileged(Native Method)
+ at javax.security.auth.Subject.doAs(Subject.java:415)
+ at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)
+ at org.apache.accumulo.core.rpc.UGIAssumingTransport.open(UGIAssumingTransport.java:49)
+ at org.apache.accumulo.core.rpc.ThriftUtil.createClientTransport(ThriftUtil.java:357)
+ at org.apache.accumulo.core.rpc.ThriftUtil.createTransport(ThriftUtil.java:255)
+ at org.apache.accumulo.server.master.LiveTServerSet$TServerConnection.getTableMap(LiveTServerSet.java:106)
+ at org.apache.accumulo.master.Master.gatherTableInformation(Master.java:996)
+ at org.apache.accumulo.master.Master.access$600(Master.java:160)
+ at org.apache.accumulo.master.Master$StatusThread.updateStatus(Master.java:911)
+ at org.apache.accumulo.master.Master$StatusThread.run(Master.java:901)
+Caused by: GSSException: No valid credentials provided (Mechanism level: Server not found in Kerberos database (7) - LOOKING_UP_SERVER)
+ at sun.security.jgss.krb5.Krb5Context.initSecContext(Krb5Context.java:710)
+ at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:248)
+ at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
+ at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:193)
+ ... 16 more
+Caused by: KrbException: Server not found in Kerberos database (7) - LOOKING_UP_SERVER
+ at sun.security.krb5.KrbTgsRep.<init>(KrbTgsRep.java:73)
+ at sun.security.krb5.KrbTgsReq.getReply(KrbTgsReq.java:192)
+ at sun.security.krb5.KrbTgsReq.sendAndGetCreds(KrbTgsReq.java:203)
+ at sun.security.krb5.internal.CredentialsUtil.serviceCreds(CredentialsUtil.java:309)
+ at sun.security.krb5.internal.CredentialsUtil.acquireServiceCreds(CredentialsUtil.java:115)
+ at sun.security.krb5.Credentials.acquireServiceCreds(Credentials.java:454)
+ at sun.security.jgss.krb5.Krb5Context.initSecContext(Krb5Context.java:641)
+ ... 19 more
+Caused by: KrbException: Identifier doesn't match expected value (906)
+ at sun.security.krb5.internal.KDCRep.init(KDCRep.java:143)
+ at sun.security.krb5.internal.TGSRep.init(TGSRep.java:66)
+ at sun.security.krb5.internal.TGSRep.<init>(TGSRep.java:61)
+ at sun.security.krb5.KrbTgsRep.<init>(KrbTgsRep.java:55)
+ ... 25 more
+</code></pre>
+</div>
+
+<p>or</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>2015-01-12 14:47:29,440 [server.TThreadPoolServer] ERROR: Error occurred during processing of message.
+java.lang.RuntimeException: org.apache.thrift.transport.TTransportException: Peer indicated failure: GSS initiate failed
+ at org.apache.thrift.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:219)
+ at org.apache.accumulo.core.rpc.UGIAssumingTransportFactory$1.run(UGIAssumingTransportFactory.java:51)
+ at org.apache.accumulo.core.rpc.UGIAssumingTransportFactory$1.run(UGIAssumingTransportFactory.java:48)
+ at java.security.AccessController.doPrivileged(Native Method)
+ at javax.security.auth.Subject.doAs(Subject.java:356)
+ at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1608)
+ at org.apache.accumulo.core.rpc.UGIAssumingTransportFactory.getTransport(UGIAssumingTransportFactory.java:48)
+ at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:208)
+ at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
+ at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
+ at java.lang.Thread.run(Thread.java:745)
+Caused by: org.apache.thrift.transport.TTransportException: Peer indicated failure: GSS initiate failed
+ at org.apache.thrift.transport.TSaslTransport.receiveSaslMessage(TSaslTransport.java:190)
+ at org.apache.thrift.transport.TSaslServerTransport.handleSaslStartMessage(TSaslServerTransport.java:125)
+ at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:253)
+ at org.apache.thrift.transport.TSaslServerTransport.open(TSaslServerTransport.java:41)
+ at org.apache.thrift.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:216)
+ ... 10 more
+</code></pre>
+</div>
+
+<p><strong>A</strong>: As previously mentioned, the hostname, and subsequently the address each Accumulo process is bound/listening
+on, is extremely important when negotiating an SASL connection. This problem commonly arises when the Accumulo
+servers are not configured to listen on the address denoted by their FQDN.</p>
+
+<p>The values in the Accumulo “hosts” files (In <code class="highlighter-rouge">accumulo/conf</code>: <code class="highlighter-rouge">masters</code>, <code class="highlighter-rouge">monitors</code>, <code class="highlighter-rouge">tservers</code>, <code class="highlighter-rouge">tracers</code>,
+and <code class="highlighter-rouge">gc</code>) should match the instance componentof the Kerberos server principal (e.g. <code class="highlighter-rouge">host</code> in <code class="highlighter-rouge">accumulo/host@EXAMPLE.COM</code>).</p>
+
+<p><strong>Q</strong>: After configuring my system for Kerberos, server processes come up normally and I can interact with the system. However,
+when I attempt to use the “Recent Traces” page on the Monitor UI I get a stacktrace similar to:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>java.lang.AssertionError: AuthenticationToken should not be null
+ at org.apache.accumulo.monitor.servlets.trace.Basic.getScanner(Basic.java:139)
+ at org.apache.accumulo.monitor.servlets.trace.Summary.pageBody(Summary.java:164)
+ at org.apache.accumulo.monitor.servlets.BasicServlet.doGet(BasicServlet.java:63)
+ at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
+ at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
+ at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:738)
+ at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:551)
+ at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
+ at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:568)
+ at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
+ at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1111)
+ at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:478)
+ at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:183)
+ at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1045)
+ at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
+ at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
+ at org.eclipse.jetty.server.Server.handle(Server.java:462)
+ at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:279)
+ at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:232)
+ at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:534)
+ at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:607)
+ at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:536)
+ at java.lang.Thread.run(Thread.java:745)
+</code></pre>
+</div>
+
+<p><strong>A</strong>: This indicates that the Monitor has not been able to successfully log in a client-side user to read from the <code class="highlighter-rouge">trace</code> table. Accumulo allows the TraceServer to rely on the property <code class="highlighter-rouge">general.kerberos.keytab</code> as a fallback when logging in the trace user if the <code class="highlighter-rouge">trace.token.property.keytab</code> property isn’t defined. Some earlier versions of Accumulo did not do this same fallback for the Monitor’s use of the trace user. The end result is that if you configure <code class="highlighter-rouge">general.kerberos.keytab</code> and not <code class="highlighter-rouge">trace.token.property.keytab</code> you will end up with a system that properly logs trace information but can’t view it.</p>
+
+<p>Ensure you have set <code class="highlighter-rouge">trace.token.property.keytab</code> to point to a keytab for the principal defined in <code class="highlighter-rouge">trace.user</code> in the <code class="highlighter-rouge">accumulo-site.xml</code> file for the Monitor, since that should work in all versions of Accumulo.</p>
+
+
+ <div class="row" style="margin-top: 20px;">
+ <div class="col-md-10"><strong>Find documentation for all releases in the <a href="/docs-archive">archive</strong></div>
+ <div class="col-md-2"><a class="pull-right" href="https://github.com/apache/accumulo-website/edit/master/_docs-unreleased/administration/kerberos.md" role="button"><i class="glyphicon glyphicon-pencil"></i> <small>Edit this page</small></a></div>
+ </div>
+ </div>
+</div>
+
+ </div>
+
+
+<footer>
+
+ <p><a href="https://www.apache.org/foundation/contributing"><img src="https://www.apache.org/images/SupportApache-small.png" alt="Support the ASF" id="asf-logo" height="100" /></a></p>
+
+ <p>Copyright © 2011-2017 The Apache Software Foundation. Licensed under the <a href="https://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.</p>
+
+</footer>
+
+
+ </div>
+ </div>
+ </div>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/accumulo-website/blob/eab65f94/docs/unreleased/administration/monitoring-metrics.html
----------------------------------------------------------------------
diff --git a/docs/unreleased/administration/monitoring-metrics.html b/docs/unreleased/administration/monitoring-metrics.html
new file mode 100644
index 0000000..2563d67
--- /dev/null
+++ b/docs/unreleased/administration/monitoring-metrics.html
@@ -0,0 +1,449 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<link href="https://maxcdn.bootstrapcdn.com/bootswatch/3.3.7/paper/bootstrap.min.css" rel="stylesheet" integrity="sha384-awusxf8AUojygHf2+joICySzB780jVvQaVCAt1clU3QsyAitLGul28Qxb2r1e5g+" crossorigin="anonymous">
+<link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.css">
+<link href="/css/accumulo.css" rel="stylesheet" type="text/css">
+
+<title>Accumulo Documentation - Monitoring & Metrics</title>
+
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script>
+<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
+<script type="text/javascript" src="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.js"></script>
+<script>
+ // show location of canonical site if not currently on the canonical site
+ $(function() {
+ var host = window.location.host;
+ if (typeof host !== 'undefined' && host !== 'accumulo.apache.org') {
+ $('#non-canonical').show();
+ }
+ });
+
+ $(function() {
+ // decorate section headers with anchors
+ return $("h2, h3, h4, h5, h6").each(function(i, el) {
+ var $el, icon, id;
+ $el = $(el);
+ id = $el.attr('id');
+ icon = '<i class="fa fa-link"></i>';
+ if (id) {
+ return $el.append($("<a />").addClass("header-link").attr("href", "#" + id).html(icon));
+ }
+ });
+ });
+
+ // fix sidebar width in documentation
+ $(function() {
+ var $affixElement = $('div[data-spy="affix"]');
+ $affixElement.width($affixElement.parent().width());
+ });
+
+ // configure Google Analytics
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ if (ga.hasOwnProperty('loaded') && ga.loaded === true) {
+ ga('create', 'UA-50934829-1', 'apache.org');
+ ga('send', 'pageview');
+ }
+</script>
+
+</head>
+<body style="padding-top: 100px">
+
+ <nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="/"><img id="nav-logo" alt="Apache Accumulo" class="img-responsive" src="/images/accumulo-logo.png" width="200"
+ /></a>
+ </div>
+ <div class="collapse navbar-collapse" id="navbar-items">
+ <ul class="nav navbar-nav">
+ <li class="nav-link"><a href="/downloads">Download</a></li>
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Releases<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/release/accumulo-1.8.1/">1.8.1 (Latest)</a></li>
+ <li><a href="/release/accumulo-1.7.3/">1.7.3</a></li>
+ <li><a href="/release/accumulo-1.6.6/">1.6.6</a></li>
+ <li><a href="/release/">Archive</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/1.8/accumulo_user_manual.html">User Manual (1.8)</a></li>
+ <li><a href="/1.8/apidocs">Javadocs (1.8)</a></li>
+ <li><a href="/1.8/examples">Examples (1.8)</a></li>
+ <li><a href="/features">Features</a></li>
+ <li><a href="/glossary">Glossary</a></li>
+ <li><a href="/external-docs">External Docs</a></li>
+ <li><a href="/docs-archive/">Archive</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Community<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/get_involved">Get Involved</a></li>
+ <li><a href="/mailing_list">Mailing Lists</a></li>
+ <li><a href="/people">People</a></li>
+ <li><a href="/related-projects">Related Projects</a></li>
+ <li><a href="/contributor/">Contributor Guide</a></li>
+ </ul>
+ </li>
+ </ul>
+ <ul class="nav navbar-nav navbar-right">
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Apache Software Foundation<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="https://www.apache.org">Apache Homepage <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/licenses/LICENSE-2.0">License <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/foundation/sponsorship">Sponsorship <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/security">Security <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/foundation/thanks">Thanks <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/foundation/policies/conduct">Code of Conduct <i class="fa fa-external-link"></i></a></li>
+ </ul>
+ </li>
+ </ul>
+ </div>
+ </div>
+</nav>
+
+ <div class="container">
+ <div class="row">
+ <div class="col-md-12">
+
+ <div id="non-canonical" style="display: none; background-color: #F0E68C; padding-left: 1em;">
+ Visit the official site at: <a href="https://accumulo.apache.org">https://accumulo.apache.org</a>
+ </div>
+ <div id="content">
+
+ <div class="row">
+ <div class="col-md-3">
+ <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true" data-spy="affix">
+ <div class="panel panel-default">
+
+
+
+
+
+
+
+
+
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsegetting-started" aria-expanded="false" aria-controls="collapsegetting-started">
+ Getting started
+ </a>
+ </h4>
+ </div>
+ <div id="collapsegetting-started" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/design">Accumulo Design</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/quick-install">Quick Installation</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/clients">Accumulo Clients</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/shell">Accumulo Shell</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/table_design">Table Design</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/table_configuration">Table Configuration</a></div>
+
+ </div>
+ </div>
+
+
+
+
+
+
+
+
+
+
+
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsedevelopment" aria-expanded="false" aria-controls="collapsedevelopment">
+ Development
+ </a>
+ </h4>
+ </div>
+ <div id="collapsedevelopment" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/iterators">Iterators</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/mapreduce">MapReduce</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/proxy">Proxy</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/development_tools">Development Tools</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/sampling">Sampling</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/summaries">Summary Statistics</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/security">Security</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/high_speed_ingest">High-Speed Ingest</a></div>
+
+ </div>
+ </div>
+
+
+
+
+
+
+
+
+
+
+
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapseadministration" aria-expanded="true" aria-controls="collapseadministration">
+ Administration
+ </a>
+ </h4>
+ </div>
+ <div id="collapseadministration" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/in-depth-install">In-depth Installation</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/configuration-management">Configuration Management</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/configuration-properties">Configuration Properties</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/monitoring-metrics">Monitoring & Metrics</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/tracing">Tracing</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/fate">FATE</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/multivolume">Multi-Volume Installations</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/ssl">SSL</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/kerberos">Kerberos</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/replication">Replication</a></div>
+
+ </div>
+ </div>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsetroubleshooting" aria-expanded="false" aria-controls="collapsetroubleshooting">
+ Troubleshooting
+ </a>
+ </h4>
+ </div>
+ <div id="collapsetroubleshooting" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/basic">Basic Troubleshooting</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/advanced">Advanced Troubleshooting</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/tools">Troubleshooting Tools</a></div>
+
+ <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/system-metadata-tables">System Metadata Tables</a></div>
+
+ </div>
+ </div>
+
+
+
+ </div>
+ </div>
+ </div>
+ <div class="col-md-9">
+
+ <p><a href="/docs/unreleased/">Accumulo unreleased docs</a> >> Administration >> Monitoring & Metrics</p>
+
+
+
+ <div class="alert alert-danger" style="margin-bottom: 0px;" role="alert">This documentation is for a future release of Accumulo! <a href="/1.8/accumulo_user_manual.html">View documentation for the latest release</a>.</div>
+
+ <div class="row">
+ <div class="col-md-10"><h1>Monitoring & Metrics</h1></div>
+ <div class="col-md-2"><a class="pull-right" style="margin-top: 25px;" href="https://github.com/apache/accumulo-website/edit/master/_docs-unreleased/administration/monitoring-metrics.md" role="button"><i class="glyphicon glyphicon-pencil"></i> <small>Edit this page</small></a></div>
+ </div>
+
+ <h2 id="monitoring">Monitoring</h2>
+
+<h3 id="accumulo-monitor">Accumulo Monitor</h3>
+
+<p>The Accumulo Monitor provides an interface for monitoring the status and health of
+Accumulo components. The Accumulo Monitor provides a web UI for accessing this information at
+<code class="highlighter-rouge">http://_monitorhost_:9995/</code>.</p>
+
+<p>Things highlighted in yellow may be in need of attention.
+If anything is highlighted in red on the monitor page, it is something that definitely needs attention.</p>
+
+<p>The Overview page contains some summary information about the Accumulo instance, including the version, instance name, and instance ID.
+There is a table labeled Accumulo Master with current status, a table listing the active Zookeeper servers, and graphs displaying various metrics over time.
+These include ingest and scan performance and other useful measurements.</p>
+
+<p>The Master Server, Tablet Servers, and Tables pages display metrics grouped in different ways (e.g. by tablet server or by table).
+Metrics typically include number of entries (key/value pairs), ingest and query rates.
+The number of running scans, major and minor compactions are in the form <em>number_running</em> (<em>number_queued</em>).
+Another important metric is hold time, which is the amount of time a tablet has been waiting but unable to flush its memory in a minor compaction.</p>
+
+<p>The Server Activity page graphically displays tablet server status, with each server represented as a circle or square.
+Different metrics may be assigned to the nodes’ color and speed of oscillation.
+The Overall Avg metric is only used on the Server Activity page, and represents the average of all the other metrics (after normalization).
+Similarly, the Overall Max metric picks the metric with the maximum normalized value.</p>
+
+<p>The Garbage Collector page displays a list of garbage collection cycles, the number of files found of each type (including deletion candidates in use and files actually deleted), and the length of the deletion cycle.
+The Traces page displays data for recent traces performed (see the following section for information on <a href="/docs/unreleased/administration/tracing">tracing</a>).
+The Recent Logs page displays warning and error logs forwarded to the monitor from all Accumulo processes.
+Also, the XML and JSON links provide metrics in XML and JSON formats, respectively.</p>
+
+<h3 id="ssl">SSL</h3>
+
+<p>SSL may be enabled for the monitor page by setting the following properties in the <code class="highlighter-rouge">accumulo-site.xml</code> file:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>monitor.ssl.keyStore
+monitor.ssl.keyStorePassword
+monitor.ssl.trustStore
+monitor.ssl.trustStorePassword
+</code></pre>
+</div>
+
+<p>If the Accumulo conf directory has been configured (in particular the <code class="highlighter-rouge">accumulo-env.sh</code> file must be set up), the
+<code class="highlighter-rouge">accumulo-util gen-monitor-cert</code> command can be used to create the keystore and truststore files with random passwords. The command
+will print out the properties that need to be added to the <code class="highlighter-rouge">accumulo-site.xml</code> file. The stores can also be generated manually with the
+Java <code class="highlighter-rouge">keytool</code> command, whose usage can be seen in the <code class="highlighter-rouge">accumulo-util</code> script.</p>
+
+<p>If desired, the SSL ciphers allowed for connections can be controlled via the following properties in <code class="highlighter-rouge">accumulo-site.xml</code>:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>monitor.ssl.include.ciphers
+monitor.ssl.exclude.ciphers
+</code></pre>
+</div>
+
+<p>If SSL is enabled, the monitor URL can only be accessed via https.
+This also allows you to access the Accumulo shell through the monitor page.
+The left navigation bar will have a new link to Shell.
+An Accumulo user name and password must be entered for access to the shell.</p>
+
+<h2 id="metrics">Metrics</h2>
+
+<p>Accumulo can expose metrics through a legacy metrics library and using the Hadoop Metrics2 library.</p>
+
+<h3 id="legacy-metrics">Legacy Metrics</h3>
+
+<p>Accumulo has a legacy metrics library that can be exposes metrics using JMX endpoints or file-based logging. These metrics can
+be enabled by setting <code class="highlighter-rouge">general.legacy.metrics</code> to <code class="highlighter-rouge">true</code> in <code class="highlighter-rouge">accumulo-site.xml</code> and placing the <code class="highlighter-rouge">accumulo-metrics.xml</code>
+configuration file on the classpath (which is typically done by placing the file in the <code class="highlighter-rouge">conf/</code> directory). A template for
+<code class="highlighter-rouge">accumulo-metrics.xml</code> can be found in <code class="highlighter-rouge">conf/templates</code> of the Accumulo tarball.</p>
+
+<h3 id="hadoop-metrics2">Hadoop Metrics2</h3>
+
+<p>Hadoop Metrics2 is a library which allows for routing of metrics generated by registered MetricsSources to
+configured MetricsSinks. Examples of sinks that are implemented by Hadoop include file-based logging, Graphite and Ganglia.
+All metric sources are exposed via JMX when using Metrics2.</p>
+
+<p>Metrics2 is configured by examining the classpath for a file that matches <code class="highlighter-rouge">hadoop-metrics2*.properties</code>. The Accumulo tarball
+contains an example <code class="highlighter-rouge">hadoop-metrics2-accumulo.properties</code> file in <code class="highlighter-rouge">conf/templates</code> which can be copied to <code class="highlighter-rouge">conf/</code> to place
+on classpath. This file is used to enable file, Graphite or Ganglia sinks (some minimal configuration required for Graphite
+and Ganglia). Because the Hadoop configuration is also on the Accumulo classpath, be sure that you do not have multiple
+Metrics2 configuration files. It is recommended to consolidate metrics in a single properties file in a central location to
+remove ambiguity. The contents of <code class="highlighter-rouge">hadoop-metrics2-accumulo.properties</code> can be added to a central <code class="highlighter-rouge">hadoop-metrics2.properties</code>
+in <code class="highlighter-rouge">$HADOOP_CONF_DIR</code>.</p>
+
+<p>As a note for configuring the file sink, the provided path should be absolute. A relative path or file name will be created relative
+to the directory in which the Accumulo process was started. External tools, such as logrotate, can be used to prevent these files
+from growing without bound.</p>
+
+<p>Each server process should have log messages from the Metrics2 library about the sinks that were created. Be sure to check
+the Accumulo processes log files when debugging missing metrics output.</p>
+
+<p>For additional information on configuring Metrics2, visit the <a href="https://hadoop.apache.org/docs/current/api/org/apache/hadoop/metrics2/package-summary.html">Javadoc page for Metrics2</a>.</p>
+
+
+
+ <div class="row" style="margin-top: 20px;">
+ <div class="col-md-10"><strong>Find documentation for all releases in the <a href="/docs-archive">archive</strong></div>
+ <div class="col-md-2"><a class="pull-right" href="https://github.com/apache/accumulo-website/edit/master/_docs-unreleased/administration/monitoring-metrics.md" role="button"><i class="glyphicon glyphicon-pencil"></i> <small>Edit this page</small></a></div>
+ </div>
+ </div>
+</div>
+
+ </div>
+
+
+<footer>
+
+ <p><a href="https://www.apache.org/foundation/contributing"><img src="https://www.apache.org/images/SupportApache-small.png" alt="Support the ASF" id="asf-logo" height="100" /></a></p>
+
+ <p>Copyright © 2011-2017 The Apache Software Foundation. Licensed under the <a href="https://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.</p>
+
+</footer>
+
+
+ </div>
+ </div>
+ </div>
+</body>
+</html>