You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/04/28 03:53:52 UTC
svn commit: r19317 [10/17] - in /dev/incubator/metron/0.4.0-RC2: ./
book-site/ book-site/css/ book-site/images/ book-site/images/logos/
book-site/images/profiles/ book-site/img/ book-site/js/
book-site/metron-analytics/ book-site/metron-analytics/metro...
Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/Kerberos-setup.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/Kerberos-setup.html (added)
+++ dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/Kerberos-setup.html Fri Apr 28 03:53:51 2017
@@ -0,0 +1,726 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-27
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170427" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Kerberos Setup</title>
+ <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../../css/site.css" />
+ <link rel="stylesheet" href="../../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="http://metron.apache.org/" id="bannerLeft">
+ <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org" class="externalLink" title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+ Metron</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Kerberos Setup</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-analytics/index.html" title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/index.html" title="Deployment">
+ <i class="icon-chevron-down"></i>
+ Deployment</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup">
+ <i class="none"></i>
+ Kerberos-manual-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+ <i class="none"></i>
+ Amazon-ec2</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+ <i class="none"></i>
+ Ansible-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+ <i class="none"></i>
+ Rpm-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+ <i class="none"></i>
+ Packer-build</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/roles/index.html" title="Roles">
+ <i class="icon-chevron-right"></i>
+ Roles</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/index.html" title="Vagrant">
+ <i class="icon-chevron-down"></i>
+ Vagrant</a>
+ <ul class="nav nav-list">
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Kerberos-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform">
+ <i class="none"></i>
+ Codelab-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/fastcapa-test-platform/index.html" title="Fastcapa-test-platform">
+ <i class="none"></i>
+ Fastcapa-test-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/full-dev-platform/index.html" title="Full-dev-platform">
+ <i class="none"></i>
+ Full-dev-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/quick-dev-platform/index.html" title="Quick-dev-platform">
+ <i class="none"></i>
+ Quick-dev-platform</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+
+ <a href="../../metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-interface/metron-config/index.html" title="Config">
+ <i class="none"></i>
+ Config</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-interface/metron-rest/index.html" title="Rest">
+ <i class="none"></i>
+ Rest</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/index.html" title="Platform">
+ <i class="icon-chevron-right"></i>
+ Platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-sensors/index.html" title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Kerberos Setup</h1>
+<p>This document provides instructions for kerberizing Metron’s Vagrant-based development environments; “Quick Dev” and “Full Dev”. These instructions do not cover the Ambari MPack or sensors. General Kerberization notes can be found in the metron-deployment <a href="../index.html">README.md</a>.</p>
+
+<ul>
+
+<li><a href="#Setup">Setup</a></li>
+
+<li><a href="#Create_a_KDC">Create a KDC</a></li>
+
+<li><a href="#Enable_Kerberos">Enable Kerberos</a></li>
+
+<li><a href="#Kafka_Authorization">Kafka Authorization</a></li>
+
+<li><a href="#HBase_Authorization">HBase Authorization</a></li>
+
+<li><a href="#Storm_Authorization">Storm Authorization</a></li>
+
+<li><a href="#Start_Metron">Start Metron</a></li>
+</ul>
+<div class="section">
+<h2><a name="Setup"></a>Setup</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Deploy a Vagrant development environment; either <a href="full-dev-platform/index.html">Full Dev</a> or <a href="quick-dev-platform/index.html">Quick Dev</a>.</p></li>
+
+<li>
+<p>Export the following environment variables. These need to be set for the remainder of the instructions. Replace <tt>node1</tt> with the appropriate hosts, if you are running Metron anywhere other than Vagrant. </p>
+<p>``` export ZOOKEEPER=node1:2181 export ELASTICSEARCH=node1:9200 export KAFKA=node1:6667</p>
+<p>export HDP_HOME=“/usr/hdp/current” export KAFKA_HOME=“${HDP_HOME}/kafka-broker” export METRON_VERSION=“0.4.0” export METRON_HOME=“/usr/metron/${METRON_VERSION}” ```</p></li>
+
+<li>
+<p>Execute the following commands as root.</p>
+
+<div class="source">
+<div class="source">
+<pre>sudo su -
+</pre></div></div></li>
+
+<li>
+<p>Stop all Metron topologies. They will be restarted again once Kerberos has been enabled.</p>
+
+<div class="source">
+<div class="source">
+<pre> for topology in bro snort enrichment indexing; do
+ storm kill $topology;
+ done
+</pre></div></div></li>
+
+<li>
+<p>Create the <tt>metron</tt> user’s home directory in HDFS.</p>
+
+<div class="source">
+<div class="source">
+<pre> sudo -u hdfs hdfs dfs -mkdir /user/metron
+ sudo -u hdfs hdfs dfs -chown metron:hdfs /user/metron
+ sudo -u hdfs hdfs dfs -chmod 770 /user/metron
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h2><a name="Create_a_KDC"></a>Create a KDC</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Install dependencies.</p>
+
+<div class="source">
+<div class="source">
+<pre> yum -y install krb5-server krb5-libs krb5-workstation
+</pre></div></div></li>
+
+<li>
+<p>Define the host, <tt>node1</tt>, as the KDC. </p>
+
+<div class="source">
+<div class="source">
+<pre> sed -i 's/kerberos.example.com/node1/g' /etc/krb5.conf
+ cp -f /etc/krb5.conf /var/lib/ambari-server/resources/scripts
+</pre></div></div></li>
+
+<li>
+<p>Do not copy/paste this full set of commands as the <tt>kdb5_util</tt> command will not run as expected. Run the commands individually to ensure they all execute. This step takes a moment. It creates the kerberos database.</p>
+<p>``` kdb5_util create -s</p></li>
+</ol>
+<p> /etc/rc.d/init.d/krb5kdc start chkconfig krb5kdc on</p>
+<p> /etc/rc.d/init.d/kadmin start chkconfig kadmin on ```</p>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Setup the <tt>admin</tt> and <tt>metron</tt> principals. You’ll <tt>kinit</tt> as the <tt>metron</tt> principal when running topologies. Make sure to remember the passwords.</p>
+
+<div class="source">
+<div class="source">
+<pre> kadmin.local -q "addprinc admin/admin"
+ kadmin.local -q "addprinc metron"
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h2><a name="Enable_Kerberos"></a>Enable Kerberos</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>In <a class="externalLink" href="http://node1:8080">Ambari</a>, setup Storm to use Kerberos and run worker jobs as the submitting user.</p>
+<p>a. Add the following properties to the custom storm-site:</p>
+
+<div class="source">
+<div class="source">
+<pre>topology.auto-credentials=['org.apache.storm.security.auth.kerberos.AutoTGT']
+nimbus.credential.renewers.classes=['org.apache.storm.security.auth.kerberos.AutoTGT']
+supervisor.run.worker.as.user=true
+</pre></div></div>
+<p>b. In the Storm config section in Ambari, choose “Add Property” under custom storm-site:</p>
+<p><img src="../readme-images/ambari-storm-site.png" alt="custom storm-site" /></p>
+<p>c. In the dialog window, choose the “bulk property add mode” toggle button and add the below values:</p>
+<p><img src="../readme-images/ambari-storm-site-properties.png" alt="custom storm-site properties" /></p></li>
+
+<li>
+<p>Kerberize the cluster via Ambari. More detailed documentation can be found <a class="externalLink" href="http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.5.3/bk_security/content/_enabling_kerberos_security_in_ambari.html">here</a>.</p>
+<p>a. For this exercise, choose existing MIT KDC (this is what we setup and installed in the previous steps.)</p>
+<p><img src="../readme-images/enable-kerberos.png" alt="enable keberos" /></p>
+<p><img src="../readme-images/enable-kerberos-started.png" alt="enable keberos get started" /></p>
+<p>b. Setup Kerberos configuration. Realm is EXAMPLE.COM. The admin principal will end up as admin/admin@EXAMPLE.COM when testing the KDC. Use the password you entered during the step for adding the admin principal.</p>
+<p><img src="../readme-images/enable-kerberos-configure-kerberos.png" alt="enable keberos configure" /></p>
+<p>c. Click through to “Start and Test Services.” Let the cluster spin up, but don’t worry about starting up Metron via Ambari - we’re going to run the parsers manually against the rest of the Hadoop cluster Kerberized. The wizard will fail at starting Metron, but this is OK. Click “continue.” When you’re finished, the custom storm-site should look similar to the following:</p>
+<p><img src="../readme-images/custom-storm-site-final.png" alt="enable keberos configure" /></p></li>
+
+<li>
+<p>Create a Metron keytab</p>
+
+<div class="source">
+<div class="source">
+<pre> kadmin.local -q "ktadd -k metron.headless.keytab metron@EXAMPLE.COM"
+ cp metron.headless.keytab /etc/security/keytabs
+ chown metron:hadoop /etc/security/keytabs/metron.headless.keytab
+ chmod 440 /etc/security/keytabs/metron.headless.keytab
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h2><a name="Kafka_Authorization"></a>Kafka Authorization</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Acquire a Kerberos ticket using the <tt>metron</tt> principal.</p>
+
+<div class="source">
+<div class="source">
+<pre> kinit -kt /etc/security/keytabs/metron.headless.keytab metron@EXAMPLE.COM
+</pre></div></div></li>
+
+<li>
+<p>Create any additional Kafka topics that you will need. We need to create the topics before adding the required ACLs. The current full dev installation will deploy bro, snort, enrichments, and indexing only. For example, you may want to add a topic for ‘yaf’ telemetry.</p>
+
+<div class="source">
+<div class="source">
+<pre> ${KAFKA_HOME}/bin/kafka-topics.sh \
+ --zookeeper ${ZOOKEEPER} \
+ --create \
+ --topic yaf \
+ --partitions 1 \
+ --replication-factor 1
+</pre></div></div></li>
+
+<li>
+<p>Setup Kafka ACLs for the <tt>bro</tt>, <tt>snort</tt>, <tt>enrichments</tt>, and <tt>indexing</tt> topics. Run the same command against any additional topics that you might be using; for example <tt>yaf</tt>.</p>
+<p>``` export KERB_USER=metron</p></li>
+</ol>
+<p> for topic in bro snort enrichments indexing; do ${KAFKA_HOME}/bin/kafka-acls.sh \ –authorizer kafka.security.auth.SimpleAclAuthorizer \ –authorizer-properties zookeeper.connect=${ZOOKEEPER} \ –add \ –allow-principal User:${KERB_USER} \ –topic ${topic} done ```</p>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Setup Kafka ACLs for the consumer groups. This command sets the ACLs for Bro, Snort, YAF, Enrichments, Indexing, and the Profiler. Execute the same command for any additional Parsers that you may be running.</p>
+<p>``` export KERB_USER=metron</p></li>
+</ol>
+<p> for group in bro_parser snort_parser yaf_parser enrichments indexing profiler; do ${KAFKA_HOME}/bin/kafka-acls.sh \ –authorizer kafka.security.auth.SimpleAclAuthorizer \ –authorizer-properties zookeeper.connect=${ZOOKEEPER} \ –add \ –allow-principal User:${KERB_USER} \ –group ${group} done ```</p>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Add the <tt>metron</tt> principal to the <tt>kafka-cluster</tt> ACL.</p>
+
+<div class="source">
+<div class="source">
+<pre> ${KAFKA_HOME}/bin/kafka-acls.sh \
+ --authorizer kafka.security.auth.SimpleAclAuthorizer \
+ --authorizer-properties zookeeper.connect=${ZOOKEEPER} \
+ --add \
+ --allow-principal User:${KERB_USER} \
+ --cluster kafka-cluster
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h2><a name="HBase_Authorization"></a>HBase Authorization</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Acquire a Kerberos ticket using the <tt>hbase</tt> principal</p>
+
+<div class="source">
+<div class="source">
+<pre> kinit -kt /etc/security/keytabs/hbase.headless.keytab hbase-metron_cluster@EXAMPLE.COM
+</pre></div></div></li>
+
+<li>
+<p>Grant permissions for the HBase tables used in Metron.</p>
+
+<div class="source">
+<div class="source">
+<pre> echo "grant 'metron', 'RW', 'threatintel'" | hbase shell
+ echo "grant 'metron', 'RW', 'enrichment'" | hbase shell
+</pre></div></div></li>
+
+<li>
+<p>If you are using the Profiler, do the same for its HBase table.</p>
+
+<div class="source">
+<div class="source">
+<pre> echo "create 'profiler', 'P'" | hbase shell
+ echo "grant 'metron', 'RW', 'profiler', 'P'" | hbase shell
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h2><a name="Storm_Authorization"></a>Storm Authorization</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Switch to the <tt>metron</tt> user and acquire a Kerberos ticket for the <tt>metron</tt> principal.</p>
+
+<div class="source">
+<div class="source">
+<pre> su metron
+ kinit -kt /etc/security/keytabs/metron.headless.keytab metron@EXAMPLE.COM
+</pre></div></div></li>
+
+<li>
+<p>Create the directory <tt>/home/metron/.storm</tt> and switch to that directory.</p>
+
+<div class="source">
+<div class="source">
+<pre> mkdir /home/metron/.storm
+ cd /home/metron/.storm
+</pre></div></div></li>
+
+<li>
+<p>Create a client JAAS file at <tt>/home/metron/.storm/client_jaas.conf</tt>. This should look identical to the Storm client JAAS file located at <tt>/etc/storm/conf/client_jaas.conf</tt> except for the addition of a <tt>Client</tt> stanza. The <tt>Client</tt> stanza is used for Zookeeper. All quotes and semicolons are necessary.</p>
+
+<div class="source">
+<div class="source">
+<pre>cat << EOF > client_jaas.conf
+StormClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useTicketCache=true
+ renewTicket=true
+ serviceName="nimbus";
+};
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="/etc/security/keytabs/metron.headless.keytab"
+ storeKey=true
+ useTicketCache=false
+ serviceName="zookeeper"
+ principal="metron@EXAMPLE.COM";
+};
+KafkaClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="/etc/security/keytabs/metron.headless.keytab"
+ storeKey=true
+ useTicketCache=false
+ serviceName="kafka"
+ principal="metron@EXAMPLE.COM";
+};
+EOF
+</pre></div></div></li>
+
+<li>
+<p>Create a YAML file at <tt>/home/metron/.storm/storm.yaml</tt>. This should point to the client JAAS file. Set the array of nimbus hosts accordingly.</p>
+
+<div class="source">
+<div class="source">
+<pre>cat << EOF > /home/metron/.storm/storm.yaml
+nimbus.seeds : ['node1']
+java.security.auth.login.config : '/home/metron/.storm/client_jaas.conf'
+storm.thrift.transport : 'org.apache.storm.security.auth.kerberos.KerberosSaslTransportPlugin'
+EOF
+</pre></div></div></li>
+
+<li>
+<p>Create an auxiliary storm configuration file at <tt>/home/metron/storm-config.json</tt>. Note the login config option in the file points to the client JAAS file.</p>
+
+<div class="source">
+<div class="source">
+<pre>cat << EOF > /home/metron/storm-config.json
+{
+ "topology.worker.childopts" : "-Djava.security.auth.login.config=/home/metron/.storm/client_jaas.conf"
+}
+EOF
+</pre></div></div></li>
+
+<li>
+<p>Configure the Enrichment, Indexing and the Profiler topologies to use the client JAAS file. Add the following properties to each of the topology properties files.</p>
+
+<div class="source">
+<div class="source">
+<pre> kafka.security.protocol=PLAINTEXTSASL
+ topology.worker.childopts=-Djava.security.auth.login.config=/home/metron/.storm/client_jaas.conf
+</pre></div></div>
+
+<ul>
+
+<li><tt>${METRON_HOME}/config/enrichment.properties</tt></li>
+
+<li><tt>${METRON_HOME}/config/elasticsearch.properties</tt></li>
+
+<li><tt>${METRON_HOME}/config/profiler.properties</tt></li>
+ </ul>
+<p>Use the following command to automate this step.</p>
+
+<div class="source">
+<div class="source">
+<pre>for file in enrichment.properties elasticsearch.properties profiler.properties; do
+ echo ${file}
+ sed -i "s/^kafka.security.protocol=.*/kafka.security.protocol=PLAINTEXTSASL/" "${METRON_HOME}/config/${file}"
+ sed -i "s/^topology.worker.childopts=.*/topology.worker.childopts=-Djava.security.auth.login.config=\/home\/metron\/.storm\/client_jaas.conf/" "${METRON_HOME}/config/${file}"
+done
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h2><a name="Start_Metron"></a>Start Metron</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Switch to the <tt>metron</tt> user and acquire a Kerberos ticket for the <tt>metron</tt> principal.</p>
+
+<div class="source">
+<div class="source">
+<pre> su metron
+ kinit -kt /etc/security/keytabs/metron.headless.keytab metron@EXAMPLE.COM
+</pre></div></div></li>
+
+<li>
+<p>Restart the parser topologies. Be sure to pass in the new parameter, <tt>-ksp</tt> or <tt>--kafka_security_protocol</tt>. The following command will start only the Bro and Snort topologies. Execute the same command for any other Parsers that you may need, for example <tt>yaf</tt>. </p>
+
+<div class="source">
+<div class="source">
+<pre>for parser in bro snort; do
+ ${METRON_HOME}/bin/start_parser_topology.sh \
+ -z ${ZOOKEEPER} \
+ -s ${parser} \
+ -ksp SASL_PLAINTEXT \
+ -e /home/metron/storm-config.json;
+done
+</pre></div></div></li>
+
+<li>
+<p>Restart the Enrichment and Indexing topologies.</p>
+
+<div class="source">
+<div class="source">
+<pre> ${METRON_HOME}/bin/start_enrichment_topology.sh
+ ${METRON_HOME}/bin/start_elasticsearch_topology.sh
+</pre></div></div></li>
+
+<li>
+<p>Push some sample data to one of the parser topics. E.g for Bro we took raw data from <a href="../../metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput/index.html">incubator-metron/metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput</a></p>
+
+<div class="source">
+<div class="source">
+<pre> cat sample-bro.txt | ${KAFKA_HOME}/kafka-broker/bin/kafka-console-producer.sh \
+ --broker-list ${KAFKA} \
+ --security-protocol SASL_PLAINTEXT \
+ --topic bro
+</pre></div></div></li>
+
+<li>
+<p>Wait a few moments for data to flow through the system and then check for data in the Elasticsearch indices. Replace yaf with whichever parser type you’ve chosen.</p>
+
+<div class="source">
+<div class="source">
+<pre> curl -XGET "${ELASTICSEARCH}/bro*/_search"
+ curl -XGET "${ELASTICSEARCH}/bro*/_count"
+</pre></div></div></li>
+
+<li>
+<p>You should have data flowing from the parsers all the way through to the indexes. This completes the Kerberization instructions</p></li>
+</ol></div>
+<div class="section">
+<h2><a name="More_Information"></a>More Information</h2>
+<div class="section">
+<h3><a name="Kerberos"></a>Kerberos</h3>
+<p>Unsure of your Kerberos principal associated with a keytab? There are a couple ways to get this. One is via the list of principals that Ambari provides via downloadable csv. If you didn’t download this list, you can also check the principal manually by running the following against the keytab.</p>
+
+<div class="source">
+<div class="source">
+<pre>klist -kt /etc/security/keytabs/<keytab-file-name>
+</pre></div></div>
+<p>E.g.</p>
+
+<div class="source">
+<div class="source">
+<pre>klist -kt /etc/security/keytabs/hbase.headless.keytab
+Keytab name: FILE:/etc/security/keytabs/hbase.headless.keytab
+KVNO Timestamp Principal
+---- ----------------- --------------------------------------------------------
+ 1 03/28/17 19:29:36 hbase-metron_cluster@EXAMPLE.COM
+ 1 03/28/17 19:29:36 hbase-metron_cluster@EXAMPLE.COM
+ 1 03/28/17 19:29:36 hbase-metron_cluster@EXAMPLE.COM
+ 1 03/28/17 19:29:36 hbase-metron_cluster@EXAMPLE.COM
+ 1 03/28/17 19:29:36 hbase-metron_cluster@EXAMPLE.COM
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Kafka_with_Kerberos_enabled"></a>Kafka with Kerberos enabled</h3>
+<div class="section">
+<h4><a name="Write_data_to_a_topic_with_SASL"></a>Write data to a topic with SASL</h4>
+
+<div class="source">
+<div class="source">
+<pre>cat sample-yaf.txt | ${KAFKA_HOME}/bin/kafka-console-producer.sh \
+ --broker-list ${KAFKA} \
+ --security-protocol PLAINTEXTSASL \
+ --topic yaf
+</pre></div></div></div>
+<div class="section">
+<h4><a name="View_topic_data_from_latest_offset_with_SASL"></a>View topic data from latest offset with SASL</h4>
+
+<div class="source">
+<div class="source">
+<pre>${KAFKA_HOME}/bin/kafka-console-consumer.sh \
+ --zookeeper ${ZOOKEEPER} \
+ --security-protocol PLAINTEXTSASL \
+ --topic yaf
+</pre></div></div></div>
+<div class="section">
+<h4><a name="Modify_the_sensor-stubs_to_send_logs_via_SASL"></a>Modify the sensor-stubs to send logs via SASL</h4>
+
+<div class="source">
+<div class="source">
+<pre>sed -i 's/node1:6667 --topic/node1:6667 --security-protocol PLAINTEXTSASL --topic/' /opt/sensor-stubs/bin/start-*-stub
+for sensorstub in bro snort; do
+ service sensor-stubs stop ${sensorstub};
+ service sensor-stubs start ${sensorstub};
+done
+</pre></div></div></div></div>
+<div class="section">
+<h3><a name="References"></a>References</h3>
+
+<ul>
+
+<li><a class="externalLink" href="https://github.com/apache/storm/blob/master/SECURITY.md">https://github.com/apache/storm/blob/master/SECURITY.md</a></li>
+</ul></div></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>
Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/codelab-platform/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/codelab-platform/index.html (added)
+++ dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/codelab-platform/index.html Fri Apr 28 03:53:51 2017
@@ -0,0 +1,340 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-27
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170427" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Developer Image for Apache Metron on Virtualbox</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="http://metron.apache.org/" id="bannerLeft">
+ <img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org" class="externalLink" title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+ Metron</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../../../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Developer Image for Apache Metron on Virtualbox</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../../../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-analytics/index.html" title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/index.html" title="Deployment">
+ <i class="icon-chevron-down"></i>
+ Deployment</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup">
+ <i class="none"></i>
+ Kerberos-manual-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+ <i class="none"></i>
+ Amazon-ec2</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+ <i class="none"></i>
+ Ansible-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+ <i class="none"></i>
+ Rpm-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+ <i class="none"></i>
+ Packer-build</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/roles/index.html" title="Roles">
+ <i class="icon-chevron-right"></i>
+ Roles</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant">
+ <i class="icon-chevron-down"></i>
+ Vagrant</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup">
+ <i class="none"></i>
+ Kerberos-setup</a>
+ </li>
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Codelab-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/fastcapa-test-platform/index.html" title="Fastcapa-test-platform">
+ <i class="none"></i>
+ Fastcapa-test-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/full-dev-platform/index.html" title="Full-dev-platform">
+ <i class="none"></i>
+ Full-dev-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/quick-dev-platform/index.html" title="Quick-dev-platform">
+ <i class="none"></i>
+ Quick-dev-platform</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-interface/metron-config/index.html" title="Config">
+ <i class="none"></i>
+ Config</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-interface/metron-rest/index.html" title="Rest">
+ <i class="none"></i>
+ Rest</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-platform/index.html" title="Platform">
+ <i class="icon-chevron-right"></i>
+ Platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-sensors/index.html" title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Developer Image for Apache Metron on Virtualbox</h1>
+<p>This image is a fully functional Metron installation that has been pre-loaded with Ambari, HDP and Metron.</p>
+<div class="section">
+<h2><a name="Getting_Started"></a>Getting Started</h2>
+<div class="section">
+<h3><a name="Prerequisites"></a>Prerequisites</h3>
+<p>As with the Singlenode Full Image, the computer used to deploy Apache Metron will need to have the following components installed.</p>
+
+<ul>
+
+<li><a class="externalLink" href="https://github.com/ansible/ansible">Ansible</a> (2.0.0.2 or 2.2.2.0)</li>
+
+<li><a class="externalLink" href="https://www.vagrantup.com">Vagrant</a> 1.8.1</li>
+
+<li><a class="externalLink" href="https://virtualbox.org">Virtualbox</a> 5.0.16</li>
+
+<li>Python 2.7.11</li>
+
+<li>Maven 3.3.9</li>
+</ul></div>
+<div class="section">
+<h3><a name="Ensure_vagrant_hostmanager_is_installed"></a>Ensure vagrant hostmanager is installed</h3>
+<p>To check and make sure you have the plugin installed execute the following:</p>
+
+<div class="source">
+<div class="source">
+<pre> vagrant plugin list
+</pre></div></div>
+<p>If you have it installed you should see the following listed in the output:</p>
+
+<div class="source">
+<div class="source">
+<pre> vagrant-hostmanager (1.8.1)
+</pre></div></div>
+<p>If it is not installed, you can install it with the following command:</p>
+
+<div class="source">
+<div class="source">
+<pre> vagrant plugin install vagrant-hostmanager
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Launch_the_Metron_Development_Image"></a>Launch the Metron Development Image</h3>
+<p>Start the image with the following commands:</p>
+
+<div class="source">
+<div class="source">
+<pre> cd metron-deployment/vagrant/codelab-platform
+ ./run.sh
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Work_with_Metron"></a>Work with Metron</h3>
+<p>As you build out new capabilities for Metron, you will need to re-deploy the Storm topologies. To do so, first HALT the running Storm topologies and then run:</p>
+
+<div class="source">
+<div class="source">
+<pre>./run_enrichment_role.sh
+</pre></div></div>
+<p>Remember Navigate to the following resources to explore your newly minted Apache Metron environment.</p>
+
+<ul>
+
+<li><a class="externalLink" href="http://node1:5000">Metron</a></li>
+
+<li><a class="externalLink" href="http://node1:8080">Ambari</a></li>
+
+<li><a class="externalLink" href="http://node1:2812">Services</a></li>
+</ul>
+<p>Connecting to the host through SSH is as simple as running the following command.</p>
+
+<div class="source">
+<div class="source">
+<pre> vagrant ssh
+</pre></div></div></div></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>
Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/fastcapa-test-platform/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/fastcapa-test-platform/index.html (added)
+++ dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/fastcapa-test-platform/index.html Fri Apr 28 03:53:51 2017
@@ -0,0 +1,338 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-27
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170427" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Fastcapa Test Environment</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="http://metron.apache.org/" id="bannerLeft">
+ <img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org" class="externalLink" title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+ Metron</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../../../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Fastcapa Test Environment</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../../../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-analytics/index.html" title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/index.html" title="Deployment">
+ <i class="icon-chevron-down"></i>
+ Deployment</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup">
+ <i class="none"></i>
+ Kerberos-manual-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+ <i class="none"></i>
+ Amazon-ec2</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+ <i class="none"></i>
+ Ansible-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+ <i class="none"></i>
+ Rpm-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+ <i class="none"></i>
+ Packer-build</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/roles/index.html" title="Roles">
+ <i class="icon-chevron-right"></i>
+ Roles</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant">
+ <i class="icon-chevron-down"></i>
+ Vagrant</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup">
+ <i class="none"></i>
+ Kerberos-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform">
+ <i class="none"></i>
+ Codelab-platform</a>
+ </li>
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Fastcapa-test-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/full-dev-platform/index.html" title="Full-dev-platform">
+ <i class="none"></i>
+ Full-dev-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/quick-dev-platform/index.html" title="Quick-dev-platform">
+ <i class="none"></i>
+ Quick-dev-platform</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-interface/metron-config/index.html" title="Config">
+ <i class="none"></i>
+ Config</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-interface/metron-rest/index.html" title="Rest">
+ <i class="none"></i>
+ Rest</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-platform/index.html" title="Platform">
+ <i class="icon-chevron-right"></i>
+ Platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-sensors/index.html" title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Fastcapa Test Environment</h1>
+<p>Provides a test environment for the development and testing of Fastcapa. The environment is automatically validated after it is created to ensure that Fastcapa is behaving correctly.</p>
+<p>Two virtualized nodes are launched with Vagrant that can communicate with one another over a private network. </p>
+
+<ul>
+
+<li>The <tt>source</tt> node uses Metron’s <tt>pcap_replay</tt> functionality to transmit raw network packet data over a private network.</li>
+
+<li>The <tt>sink</tt> node is running <tt>fastcapa</tt> and is capturing these network packets.</li>
+
+<li>Fastcapa then transforms and bundles the packets into a message.</li>
+
+<li>The message is sent to a Kafka broker running on the <tt>source</tt> node.</li>
+</ul>
+<div class="section">
+<h2><a name="Getting_Started"></a>Getting Started</h2>
+<p>Simply run <tt>vagrant up</tt> to launch the environment. Automated tests are executed after provisioning completes to ensure that Fastcapa and the rest of the environment is functioning properly.</p>
+
+<div class="source">
+<div class="source">
+<pre>$ vagrant up
+==> source: Running provisioner: ansible...
+ source: Running ansible-playbook...
+...
+TASK [debug] *******************************************************************
+ok: [source] => {
+ "msg": "Successfully received packets sent from pcap-replay!"
+}
+...
+TASK [debug] *******************************************************************
+ok: [source] => {
+ "msg": "Successfully received a Kafka message from fastcapa!"
+}
+</pre></div></div></div>
+<div class="section">
+<h2><a name="Going_Deeper"></a>Going Deeper</h2>
+<p>This section will outline in more detail the environment and how to interact with it.</p>
+<div class="section">
+<h3><a name="source"></a><tt>source</tt></h3>
+<p>To validate that the <tt>source</tt> node is functioning properly, run the following commands.</p>
+<p>First, ensure that the <tt>pcap-replay</tt> service is running.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant ssh source
+sudo service pcap-replay status
+</pre></div></div>
+<p>Use <tt>tcpdump</tt> to ensure that the raw packet data is being sent over the private network. Enter ‘CTRL-C’ to kill the <tt>tcpdump</tt> process once you are able to see that packets are being sent.</p>
+
+<div class="source">
+<div class="source">
+<pre>sudo yum -y install tcpdump
+sudo tcpdump -i enp0s8
+</pre></div></div></div>
+<div class="section">
+<h3><a name="sink"></a><tt>sink</tt></h3>
+<p>Next validate that the <tt>sink</tt> is functioning properly. Run the following commands starting from the host operating system. </p>
+<p>First, ensure that the <tt>fastcapa</tt> service is running.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant ssh sink
+service fastcapa status
+</pre></div></div>
+<p>Ensure that the raw network packet data is being received by Kafka.</p>
+
+<div class="source">
+<div class="source">
+<pre>/usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic pcap
+</pre></div></div>
+<p>Enter ‘CTRL-C’ to kill the <tt>kafka-console-consumer</tt> process once you are able to see that packets are being sent. These packets will appear to be gibberish in the console. This is the raw binary network packet data after all.</p></div></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>
Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/full-dev-platform/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/full-dev-platform/index.html (added)
+++ dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/full-dev-platform/index.html Fri Apr 28 03:53:51 2017
@@ -0,0 +1,390 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-27
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170427" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Full Development Platform</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="http://metron.apache.org/" id="bannerLeft">
+ <img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org" class="externalLink" title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+ Metron</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../../../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Full Development Platform</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../../../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-analytics/index.html" title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/index.html" title="Deployment">
+ <i class="icon-chevron-down"></i>
+ Deployment</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup">
+ <i class="none"></i>
+ Kerberos-manual-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+ <i class="none"></i>
+ Amazon-ec2</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+ <i class="none"></i>
+ Ansible-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+ <i class="none"></i>
+ Rpm-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+ <i class="none"></i>
+ Packer-build</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/roles/index.html" title="Roles">
+ <i class="icon-chevron-right"></i>
+ Roles</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant">
+ <i class="icon-chevron-down"></i>
+ Vagrant</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup">
+ <i class="none"></i>
+ Kerberos-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform">
+ <i class="none"></i>
+ Codelab-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/fastcapa-test-platform/index.html" title="Fastcapa-test-platform">
+ <i class="none"></i>
+ Fastcapa-test-platform</a>
+ </li>
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Full-dev-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/quick-dev-platform/index.html" title="Quick-dev-platform">
+ <i class="none"></i>
+ Quick-dev-platform</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-interface/metron-config/index.html" title="Config">
+ <i class="none"></i>
+ Config</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-interface/metron-rest/index.html" title="Rest">
+ <i class="none"></i>
+ Rest</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-platform/index.html" title="Platform">
+ <i class="icon-chevron-right"></i>
+ Platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-sensors/index.html" title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Full Development Platform</h1>
+<p>This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running on Virtualbox. </p>
+<p>Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. To that end the vagrant file configuration defaults to disabling solr and yaf.</p>
+<div class="section">
+<h2><a name="Getting_Started"></a>Getting Started</h2>
+<div class="section">
+<h3><a name="Prerequisites"></a>Prerequisites</h3>
+<p>The computer used to deploy Apache Metron will need to have the following components installed.</p>
+
+<ul>
+
+<li><a class="externalLink" href="https://github.com/ansible/ansible">Ansible</a> (2.0.0.2 or 2.2.2.0)</li>
+
+<li><a class="externalLink" href="https://www.docker.com/community-edition">Docker</a></li>
+
+<li><a class="externalLink" href="https://www.vagrantup.com">Vagrant</a> 1.8.1</li>
+
+<li><a class="externalLink" href="https://github.com/devopsgroup-io/vagrant-hostmanager">Vagrant Hostmanager Plugin</a> <tt>vagrant plugin install vagrant-hostmanager</tt></li>
+
+<li><a class="externalLink" href="https://virtualbox.org">Virtualbox</a> 5.0.16</li>
+
+<li>Python 2.7.11</li>
+
+<li>Maven 3.3.9</li>
+</ul>
+<div class="section">
+<h4><a name="macOS"></a>macOS</h4>
+<p>Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent <a class="externalLink" href="http://brew.sh/">Homebrew</a> project.</p>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Install Homebrew by following the instructions at <a class="externalLink" href="http://brew.sh/">Homebrew</a>.</p></li>
+
+<li>
+<p>Run the following command in a terminal to install all of the required tools.</p>
+
+<div class="source">
+<div class="source">
+<pre>brew cask install vagrant virtualbox java docker
+brew install maven git
+</pre></div></div></li>
+
+<li>
+<p>Install Ansible by following the instructions <a class="externalLink" href="http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip">here</a>.</p></li>
+</ol></div></div>
+<div class="section">
+<h3><a name="Deploy_Metron"></a>Deploy Metron</h3>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Ensure that the Docker service is running.</p></li>
+
+<li>
+<p>Deploy Metron</p>
+
+<div class="source">
+<div class="source">
+<pre>cd metron-deployment/vagrant/full-dev-platform
+vagrant up
+</pre></div></div>
+<p>Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant provision
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h3><a name="Explore_Metron"></a>Explore Metron</h3>
+<p>Navigate to the following resources to explore your newly minted Apache Metron environment.</p>
+
+<ul>
+
+<li><a class="externalLink" href="http://node1:8080">Ambari</a></li>
+
+<li><a class="externalLink" href="http://node1:5000">Metron</a></li>
+
+<li><a class="externalLink" href="http://node1:2812">Services</a></li>
+</ul>
+<p>Connecting to the host through SSH is as simple as running the following command.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant ssh
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Working_with_Metron"></a>Working with Metron</h3>
+<p>In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the <tt>web</tt> role on the Vagrant image. This will install components (if necessary) and start the UI.</p>
+
+<div class="source">
+<div class="source">
+<pre>./run_ansible_role.sh web
+</pre></div></div>
+<p>or</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant --ansible-tags="web" provision
+</pre></div></div>
+<div class="section">
+<h4><a name="Using_Tags"></a>Using Tags</h4>
+<p>A collection of tags is specified as a comma separated list.</p>
+
+<div class="source">
+<div class="source">
+<pre>./run_ansible_role.sh "sensors,enrichment"
+
+</pre></div></div>
+<p>Tags are listed in the playbooks, some frequently used tags:</p>
+
+<ul>
+
+<li><tt>hdp-install</tt> - Install HDP</li>
+
+<li><tt>hdp-deploy</tt> - Deploy and Start HDP Services (will start all Hadoop Services)</li>
+
+<li><tt>sensors</tt> - Deploy and Start Sensors.</li>
+
+<li><tt>enrichment</tt> - Deploy and Start Enrichment Topology.</li>
+</ul>
+<p>Note also that there is a convenience script <tt>./run_enrichment_role.sh</tt> which executes Vagrant with the <tt>enrichment</tt> tag.</p></div></div></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>
Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/index.html (added)
+++ dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/index.html Fri Apr 28 03:53:51 2017
@@ -0,0 +1,282 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-27
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170427" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Vagrant Deployment</title>
+ <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../../css/site.css" />
+ <link rel="stylesheet" href="../../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="http://metron.apache.org/" id="bannerLeft">
+ <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org" class="externalLink" title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+ Metron</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Vagrant Deployment</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-analytics/index.html" title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/index.html" title="Deployment">
+ <i class="icon-chevron-down"></i>
+ Deployment</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup">
+ <i class="none"></i>
+ Kerberos-manual-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+ <i class="none"></i>
+ Amazon-ec2</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+ <i class="none"></i>
+ Ansible-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+ <i class="none"></i>
+ Rpm-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+ <i class="none"></i>
+ Packer-build</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/roles/index.html" title="Roles">
+ <i class="icon-chevron-right"></i>
+ Roles</a>
+ </li>
+
+ <li class="active">
+
+ <a href="#"><i class="icon-chevron-down"></i>Vagrant</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup">
+ <i class="none"></i>
+ Kerberos-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform">
+ <i class="none"></i>
+ Codelab-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/fastcapa-test-platform/index.html" title="Fastcapa-test-platform">
+ <i class="none"></i>
+ Fastcapa-test-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/full-dev-platform/index.html" title="Full-dev-platform">
+ <i class="none"></i>
+ Full-dev-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/vagrant/quick-dev-platform/index.html" title="Quick-dev-platform">
+ <i class="none"></i>
+ Quick-dev-platform</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+
+ <a href="../../metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-interface/metron-config/index.html" title="Config">
+ <i class="none"></i>
+ Config</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-interface/metron-rest/index.html" title="Rest">
+ <i class="none"></i>
+ Rest</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/index.html" title="Platform">
+ <i class="icon-chevron-right"></i>
+ Platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-sensors/index.html" title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Vagrant Deployment</h1>
+<p><a name="Vagrant_Deployment"></a></p>
+
+<ul>
+
+<li>Kerberos Setup</li>
+
+<li>Codelab Platform</li>
+
+<li>Fast CAPA Test Platform</li>
+
+<li>Full Dev Platform</li>
+
+<li>Quick Dev Platform</li>
+</ul>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>