You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/04/28 03:53:52 UTC
svn commit: r19317 [11/17] - in /dev/incubator/metron/0.4.0-RC2: ./
book-site/ book-site/css/ book-site/images/ book-site/images/logos/
book-site/images/profiles/ book-site/img/ book-site/js/
book-site/metron-analytics/ book-site/metron-analytics/metro...
Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/quick-dev-platform/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/quick-dev-platform/index.html (added)
+++ dev/incubator/metron/0.4.0-RC2/book-site/metron-deployment/vagrant/quick-dev-platform/index.html Fri Apr 28 03:53:51 2017
@@ -0,0 +1,363 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-27
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170427" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Quick Development Platform</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="http://metron.apache.org/" id="bannerLeft">
+ <img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org" class="externalLink" title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+ Metron</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../../../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Quick Development Platform</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../../../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-analytics/index.html" title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/index.html" title="Deployment">
+ <i class="icon-chevron-down"></i>
+ Deployment</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup">
+ <i class="none"></i>
+ Kerberos-manual-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+ <i class="none"></i>
+ Amazon-ec2</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+ <i class="none"></i>
+ Ansible-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+ <i class="none"></i>
+ Rpm-docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+ <i class="none"></i>
+ Packer-build</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/roles/index.html" title="Roles">
+ <i class="icon-chevron-right"></i>
+ Roles</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant">
+ <i class="icon-chevron-down"></i>
+ Vagrant</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup">
+ <i class="none"></i>
+ Kerberos-setup</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform">
+ <i class="none"></i>
+ Codelab-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/fastcapa-test-platform/index.html" title="Fastcapa-test-platform">
+ <i class="none"></i>
+ Fastcapa-test-platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-deployment/vagrant/full-dev-platform/index.html" title="Full-dev-platform">
+ <i class="none"></i>
+ Full-dev-platform</a>
+ </li>
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Quick-dev-platform</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-interface/metron-config/index.html" title="Config">
+ <i class="none"></i>
+ Config</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-interface/metron-rest/index.html" title="Rest">
+ <i class="none"></i>
+ Rest</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-platform/index.html" title="Platform">
+ <i class="icon-chevron-right"></i>
+ Platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../../metron-sensors/index.html" title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Quick Development Platform</h1>
+<p>This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running on Virtualbox. </p>
+<p>This image is designed for quick deployment of a single node Metron cluster running on Virtualbox. This platform is ideal for use by Metron developers. It uses a base image that has been pre-loaded with Ambari and HDP.</p>
+<p>Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. To that end the vagrant file configuration defaults to disabling solr and yaf.</p>
+<div class="section">
+<h2><a name="Getting_Started"></a>Getting Started</h2>
+<div class="section">
+<h3><a name="Prerequisites"></a>Prerequisites</h3>
+<p>As with the Full Development Platform (<tt>metron-deployment/vagrant/full-dev-platform</tt>), the computer used to deploy Apache Metron will need the following components installed.</p>
+
+<ul>
+
+<li><a class="externalLink" href="https://github.com/ansible/ansible">Ansible</a> (2.0.0.2 or 2.2.2.0)</li>
+
+<li><a class="externalLink" href="https://www.docker.com/community-edition">Docker</a></li>
+
+<li><a class="externalLink" href="https://www.vagrantup.com">Vagrant</a> 1.8.1</li>
+
+<li><a class="externalLink" href="https://github.com/devopsgroup-io/vagrant-hostmanager">Vagrant Hostmanager Plugin</a> <tt>vagrant plugin install vagrant-hostmanager</tt></li>
+
+<li><a class="externalLink" href="https://virtualbox.org">Virtualbox</a> 5.0.16</li>
+
+<li>Python 2.7.11</li>
+
+<li>Maven 3.3.9</li>
+</ul>
+<div class="section">
+<h4><a name="macOS"></a>macOS</h4>
+<p>Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent <a class="externalLink" href="http://brew.sh/">Homebrew</a> project.</p>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Install Homebrew by following the instructions at <a class="externalLink" href="http://brew.sh/">Homebrew</a>.</p></li>
+
+<li>
+<p>Run the following command in a terminal to install all of the required tools.</p>
+
+<div class="source">
+<div class="source">
+<pre> brew cask install vagrant virtualbox java docker
+ brew install maven git
+</pre></div></div></li>
+
+<li>
+<p>Install Ansible by following the instructions <a class="externalLink" href="http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip">here</a>.</p></li>
+</ol></div></div>
+<div class="section">
+<h3><a name="Deploy_Metron"></a>Deploy Metron</h3>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Ensure that the Docker service is running.</p></li>
+
+<li>
+<p>Deploy Metron</p>
+
+<div class="source">
+<div class="source">
+<pre>cd metron-deployment/vagrant/quick-dev-platform
+vagrant up
+</pre></div></div>
+<p>Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant provision
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h3><a name="Explore_Metron"></a>Explore Metron</h3>
+<p>Navigate to the following resources to explore your newly minted Apache Metron environment.</p>
+
+<ul>
+
+<li><a class="externalLink" href="http://node1:8080">Ambari</a></li>
+
+<li><a class="externalLink" href="http://node1:5000">Metron</a></li>
+
+<li><a class="externalLink" href="http://node1:2812">Services</a></li>
+</ul></div>
+<div class="section">
+<h3><a name="Working_with_Metron"></a>Working with Metron</h3>
+<p>As you build out new capabilities for Metron, you will need to re-deploy the Storm topologies. To do so, first HALT the running Storm topologies and then run the following command.</p>
+
+<div class="source">
+<div class="source">
+<pre>./run_enrichment_role.sh
+</pre></div></div>
+<p>Connecting to the host through SSH is as simple as running the following command.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant ssh
+</pre></div></div></div></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>
Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-docker/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC2/book-site/metron-docker/index.html (added)
+++ dev/incubator/metron/0.4.0-RC2/book-site/metron-docker/index.html Fri Apr 28 03:53:51 2017
@@ -0,0 +1,485 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-27
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170427" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Metron Docker</title>
+ <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../css/site.css" />
+ <link rel="stylesheet" href="../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="http://metron.apache.org/" id="bannerLeft">
+ <img src="../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org" class="externalLink" title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+ Metron</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Metron Docker</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../metron-analytics/index.html" title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../metron-deployment/index.html" title="Deployment">
+ <i class="icon-chevron-right"></i>
+ Deployment</a>
+ </li>
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Docker</a>
+ </li>
+
+ <li>
+
+ <a href="../metron-interface/metron-config/index.html" title="Config">
+ <i class="none"></i>
+ Config</a>
+ </li>
+
+ <li>
+
+ <a href="../metron-interface/metron-rest/index.html" title="Rest">
+ <i class="none"></i>
+ Rest</a>
+ </li>
+
+ <li>
+
+ <a href="../metron-platform/index.html" title="Platform">
+ <i class="icon-chevron-right"></i>
+ Platform</a>
+ </li>
+
+ <li>
+
+ <a href="../metron-sensors/index.html" title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Metron Docker</h1>
+<p><a name="Metron_Docker"></a></p>
+<p>Metron Docker is a <a class="externalLink" href="https://docs.docker.com/compose/overview/">Docker Compose</a> application that is intended for development and integration testing of Metron. Use this instead of Vagrant when:</p>
+
+<ul>
+
+<li>You want an environment that can be built and spun up quickly</li>
+
+<li>You need to frequently rebuild and restart services</li>
+
+<li>You only need to test, troubleshoot or develop against a subset of services</li>
+</ul>
+<p>Metron Docker includes these images that have been customized for Metron:</p>
+
+<ul>
+
+<li>Kafka (with Zookeeper)</li>
+
+<li>HBase</li>
+
+<li>Storm (with all topologies deployed)</li>
+
+<li>Elasticsearch</li>
+
+<li>Kibana</li>
+
+<li>HDFS</li>
+</ul>
+<div class="section">
+<h2><a name="Setup"></a>Setup</h2>
+<p>Install <a class="externalLink" href="https://docs.docker.com/docker-for-mac/">Docker for Mac</a> or <a class="externalLink" href="https://docs.docker.com/docker-for-windows/">Docker for Windows</a>. The following versions have been tested:</p>
+
+<ul>
+
+<li>Docker version 1.12.0</li>
+
+<li>docker-machine version 0.8.0</li>
+
+<li>docker-compose version 1.8.0</li>
+</ul>
+<p>Build Metron from the top level directory with:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_HOME
+$ mvn clean install -DskipTests
+</pre></div></div>
+<p>You are welcome to use an existing Docker host but we prefer one with more resources. You can create one of those with this script:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-docker
+$ cd $METRON_DOCKER_HOME && ./scripts/create-docker-machine.sh
+</pre></div></div>
+<p>This will create a host called “metron-machine”. Anytime you want to run Docker commands against this host, make sure you run this first to set the Docker environment variables:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ eval "$(docker-machine env metron-machine)"
+</pre></div></div>
+<p>If you wish to use a local docker-engine install, please set an environment variable BROKER_IP_ADDR to the IP address of your host machine. This cannot be the loopback address.</p></div>
+<div class="section">
+<h2><a name="Usage"></a>Usage</h2>
+<p>Navigate to the compose application root:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose/
+</pre></div></div>
+<p>The Metron Docker environment lifecycle is controlled by the <a class="externalLink" href="https://docs.docker.com/compose/reference/overview/">docker-compose</a> command. The service names can be found in the docker-compose.yml file. For example, to build and start the environment run this command:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ eval "$(docker-machine env metron-machine)"
+$ docker-compose up -d
+</pre></div></div>
+<p>After all services have started list the containers and ensure their status is ‘Up’:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker ps --format 'table {{.Names}}\t{{.Status}}'
+NAMES STATUS
+metron_storm_1 Up 5 minutes
+metron_hbase_1 Up 5 minutes
+metron_kibana_1 Up 5 minutes
+metron_kafkazk_1 Up 5 minutes
+metron_elasticsearch_1 Up 5 minutes
+</pre></div></div>
+<p>Various services are exposed through http on the Docker host. Get the host ip from the URL property:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-machine ls
+NAME ACTIVE DRIVER STATE URL SWARM DOCKER ERRORS
+metron-machine * virtualbox Running tcp://192.168.99.100:2376 v1.12.5
+</pre></div></div>
+<p>Then, assuming a host ip of <tt>192.168.99.100</tt>, the UIs and APIs are available at:</p>
+
+<ul>
+
+<li>Storm - <a class="externalLink" href="http://192.168.99.100:8080/">http://192.168.99.100:8080/</a></li>
+
+<li>HBase - <a class="externalLink" href="http://192.168.99.100:16010/">http://192.168.99.100:16010/</a></li>
+
+<li>Elasticsearch - <a class="externalLink" href="http://192.168.99.100:9200/_plugin/head/">http://192.168.99.100:9200/_plugin/head/</a></li>
+
+<li>Kibana - <a class="externalLink" href="http://192.168.99.100:5601/">http://192.168.99.100:5601/</a></li>
+
+<li>HDFS (Namenode) - <a class="externalLink" href="http://192.168.99.100:50070/">http://192.168.99.100:50070/</a></li>
+</ul>
+<p>The Storm logs can be useful when troubleshooting topologies. They can be found on the Storm container in <tt>/usr/share/apache-storm/logs</tt>.</p>
+<p>When done using the machine, shut it down with:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose down
+</pre></div></div></div>
+<div class="section">
+<h2><a name="Examples"></a>Examples</h2>
+
+<ul>
+
+<li><a href="#Deploy_a_new_parser_class">Deploy a new parser class</a></li>
+
+<li><a href="#Connect_to_a_container">Connect to a container</a></li>
+
+<li><a href="create-a-sensor-from-sample-data/index.html">Create a sensor from sample data</a></li>
+
+<li><a href="upload-configs-to-zookeeper/index.html">Upload configs to Zookeeper</a></li>
+
+<li><a href="manage-a-topology/index.html">Manage a topology</a></li>
+
+<li><a href="run-sensor-data-end-to-end/index.html">Run sensor data end to end</a></li>
+</ul>
+<div class="section">
+<h3><a name="Deploy_a_new_parser_class"></a>Deploy a new parser class</h3>
+<p>After adding a new parser to metron-parsers, build Metron from the top level directory:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_HOME
+$ mvn clean install -DskipTests
+</pre></div></div>
+<p>Then run these commands to redeploy the parsers to the Storm image:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose
+$ docker-compose down
+$ docker-compose build storm
+$ docker-compose up -d
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Connect_to_a_container"></a>Connect to a container</h3>
+<p>Suppose there is a problem with Kafka and the logs are needed for further investigation. Run this command to connect and explore the running Kafka container:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose
+$ docker-compose exec kafkazk bash
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Create_a_sensor_from_sample_data"></a>Create a sensor from sample data</h3>
+<p>A tool for producing test data in Kafka is included with the Kafka/Zookeeper image. It loops through lines in a test data file and outputs them to Kafka at the desired frequency. Create a test data file in <tt>./kafkazk/data/</tt> and rebuild the Kafka/Zookeeper image:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose
+$ printf 'first test data\nsecond test data\nthird test data\n' > ./kafkazk/data/TestData.txt
+$ docker-compose down
+$ docker-compose build kafkazk
+$ docker-compose up -d
+</pre></div></div>
+<p>This will deploy the test data file to the Kafka/Zookeeper container. Now that data can be streamed to a Kafka topic:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec kafkazk ./bin/produce-data.sh
+Usage: produce-data.sh data_path topic [message_delay_in_seconds]
+
+# Stream data in TestData.txt to the 'test' Kafka topic at a frequency of 5 seconds (default is 1 second)
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/TestData.txt test 5
+</pre></div></div>
+<p>The Kafka/Zookeeper image comes with sample Bro and Squid data:</p>
+
+<div class="source">
+<div class="source">
+<pre># Stream Bro test data every 1 second
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro
+
+# Stream Squid test data every 0.1 seconds
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/SquidExampleOutput.txt squid 0.1
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Upload_configs_to_Zookeeper"></a>Upload configs to Zookeeper</h3>
+<p>Parser configs and a global config configured for this Docker environment are included with the Kafka/Zookeeper image. Load them with:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec kafkazk bash
+# $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m PUSH -i $METRON_HOME/config/zookeeper
+# exit
+</pre></div></div>
+<p>Dump out the configs with:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec kafkazk bash
+# $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m DUMP
+# exit
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Manage_a_topology"></a>Manage a topology</h3>
+<p>The Storm image comes with a script to easily start parser topologies:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_docker_parser_topology.sh sensor_name
+</pre></div></div>
+<p>The enrichment topology can be started with:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh
+</pre></div></div>
+<p>The indexing topology can be started with:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh
+</pre></div></div>
+<p>Topologies can be stopped using the Storm CLI. For example, stop the enrichment topology with:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm storm kill enrichments -w 0
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Run_sensor_data_end_to_end"></a>Run sensor data end to end</h3>
+<p>First ensure configs were uploaded as described in the previous example. Then start a sensor and leave it running:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro
+</pre></div></div>
+<p>Open a separate console session and verify the sensor is running by consuming a message from Kafka:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-docker
+$ cd $METRON_DOCKER_HOME/compose
+$ docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic bro
+</pre></div></div>
+<p>A new message should be printed every second. Now kill the consumer and start the Bro parser topology:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec storm ./bin/start_docker_parser_topology.sh bro
+</pre></div></div>
+<p>Bro data should be flowing through the bro parser topology and into the Kafka enrichments topic. The enrichments topic should be created automatically:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec kafkazk ./bin/kafka-topics.sh --zookeeper localhost:2181 --list
+bro
+enrichments
+indexing
+</pre></div></div>
+<p>Verify parsed Bro data is in the Kafka enrichments topic:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic enrichments
+</pre></div></div>
+<p>Now start the enrichment topology:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh
+</pre></div></div>
+<p>Parsed Bro data should be flowing through the enrichment topology and into the Kafka indexing topic. Verify enriched Bro data is in the Kafka indexing topic:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic indexing
+</pre></div></div>
+<p>Now start the indexing topology:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh
+</pre></div></div>
+<p>Enriched Bro data should now be present in the Elasticsearch container:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-machine ls
+NAME ACTIVE DRIVER STATE URL SWARM DOCKER ERRORS
+metron-machine * virtualbox Running tcp://192.168.99.100:2376 v1.12.5
+
+$ curl -XGET http://192.168.99.100:9200/_cat/indices?v
+health status index pri rep docs.count docs.deleted store.size pri.store.size
+yellow open .kibana 1 1 1 0 3.1kb 3.1kb
+yellow open bro_index_2016.12.19.18 5 1 180 0 475kb 475kb
+</pre></div></div></div></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>
Added: dev/incubator/metron/0.4.0-RC2/book-site/metron-interface/metron-config/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC2/book-site/metron-interface/metron-config/index.html (added)
+++ dev/incubator/metron/0.4.0-RC2/book-site/metron-interface/metron-config/index.html Fri Apr 28 03:53:51 2017
@@ -0,0 +1,290 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-27
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170427" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Metron Management UI</title>
+ <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../../css/site.css" />
+ <link rel="stylesheet" href="../../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="http://metron.apache.org/" id="bannerLeft">
+ <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org" class="externalLink" title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+ Metron</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Metron Management UI</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2017-04-27</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-analytics/index.html" title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/index.html" title="Deployment">
+ <i class="icon-chevron-right"></i>
+ Deployment</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>Config</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-interface/metron-rest/index.html" title="Rest">
+ <i class="none"></i>
+ Rest</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/index.html" title="Platform">
+ <i class="icon-chevron-right"></i>
+ Platform</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-sensors/index.html" title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Metron Management UI</h1>
+<p><a name="Metron_Management_UI"></a></p>
+<p>This module provides a user interface for management functions in Metron.</p>
+<div class="section">
+<h2><a name="Prerequisites"></a>Prerequisites</h2>
+
+<ul>
+
+<li>A network accessible Metron REST application</li>
+
+<li>nodejs v6.9+ (nodejs can be installed on quick dev with <tt>curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - && yum install -y nodejs</tt>)</li>
+</ul></div>
+<div class="section">
+<h2><a name="Installation"></a>Installation</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Build Metron:</p>
+
+<div class="source">
+<div class="source">
+<pre>mvn clean package -DskipTests
+</pre></div></div></li>
+
+<li>
+<p>Copy <tt>incubator-metron/metron-interface/metron-config/target/metron-config-METRON_VERSION-archive.tar.gz</tt> to the desired host.</p></li>
+
+<li>
+<p>Untar the archive in the target directory. The directory structure will look like:</p>
+
+<div class="source">
+<div class="source">
+<pre>bin
+ start_management_ui.sh
+web
+ expressjs
+ package.json
+ server.js
+ management-ui
+ web assets (html, css, js, ...)
+</pre></div></div></li>
+
+<li>
+<p>For production use, the contents of the <tt>./web/management-ui</tt> directory should be deployed to a web server with paths <tt>/api/v1</tt> and <tt>/logout</tt> mapped to the REST application url. </p></li>
+
+<li>
+<p>As an example, a convenience script is included that will install a simple <a class="externalLink" href="https://github.com/expressjs/express">expressjs</a> webserver.</p></li>
+
+<li>
+<p>Then start the application with the script:</p>
+
+<div class="source">
+<div class="source">
+<pre>./bin/start_management_ui.sh
+Usage: server.js -p [port] -r [restUrl]
+Options:
+ -p Port to run metron management ui [required]
+ -r, --resturl Url where metron rest application is available [required]
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h2><a name="Usage"></a>Usage</h2>
+<p>The application will be available at <a class="externalLink" href="http://host:4200">http://host:4200</a> with credentials <tt>user/password</tt>, assuming the default port is configured and the <tt>dev</tt> profile is included when starting the REST application. See the <a href="../metron-rest/index.html#security">REST application</a> documentation for more information about security configuration for production.</p></div>
+<div class="section">
+<h2><a name="Development"></a>Development</h2>
+<p>The Management UI can also be started in development mode. This allows changes to web assets to be seen interactively.</p>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Install the application with dev dependencies:</p>
+
+<div class="source">
+<div class="source">
+<pre>npm install
+</pre></div></div></li>
+
+<li>
+<p>Start the application:</p>
+
+<div class="source">
+<div class="source">
+<pre>./scripts/start_dev.sh
+</pre></div></div></li>
+</ol>
+<p>The application will be available at <a class="externalLink" href="http://localhost:4200/">http://localhost:4200/</a>. The REST application url defaults to <tt>http://localhost:8080</tt> but can be changed in the <tt>proxy.conf.json</tt> file.</p></div>
+<div class="section">
+<h2><a name="Testing"></a>Testing</h2>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Install the application with dev dependencies:</p>
+
+<div class="source">
+<div class="source">
+<pre>npm install
+</pre></div></div></li>
+
+<li>
+<p>Unit tests can be run with:</p>
+
+<div class="source">
+<div class="source">
+<pre>npm test
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h2><a name="License"></a>License</h2>
+<p>This projects bundles Font Awesome which is available under the SIL Open Font License. See <a class="externalLink" href="http://fontawesome.io/license/">http://fontawesome.io/license/</a> for more details.</p></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>