You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/09/19 18:42:27 UTC
[04/12] metron git commit: METRON-1191 update public web site to
point at 0.4.1 new release (mattf-horton) closes apache/metron#764
http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-platform/metron-data-management/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-data-management/index.html b/site/current-book/metron-platform/metron-data-management/index.html
index d73ea2e..b04a6f4 100644
--- a/site/current-book/metron-platform/metron-data-management/index.html
+++ b/site/current-book/metron-platform/metron-data-management/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-06-27
+ | Generated by Apache Maven Doxia at 2017-09-15
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170627" />
+ <meta name="Date-Revision-yyyymmdd" content="20170915" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Resource Data Management</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
@@ -61,8 +61,8 @@
- <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li>
- <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+ <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.1</li>
</ul>
</div>
@@ -75,7 +75,7 @@
<ul class="nav nav-list">
<li class="nav-header">User Documentation</li>
-
+
<li>
<a href="../../index.html" title="Metron">
@@ -96,7 +96,14 @@
<i class="icon-chevron-right"></i>
Analytics</a>
</li>
-
+
+ <li>
+
+ <a href="../../metron-contrib/metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
<li>
<a href="../../metron-deployment/index.html" title="Deployment">
@@ -106,9 +113,9 @@
<li>
- <a href="../../metron-docker/index.html" title="Docker">
+ <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
<i class="none"></i>
- Docker</a>
+ Alerts</a>
</li>
<li>
@@ -124,7 +131,7 @@
<i class="none"></i>
Rest</a>
</li>
-
+
<li>
<a href="../../metron-platform/index.html" title="Platform">
@@ -134,17 +141,24 @@
<li>
+ <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide">
+ <i class="none"></i>
+ Performance-tuning-guide</a>
+ </li>
+
+ <li>
+
<a href="../../metron-platform/metron-api/index.html" title="Api">
<i class="none"></i>
Api</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-common/index.html" title="Common">
- <i class="icon-chevron-right"></i>
+ <i class="none"></i>
Common</a>
- </li>
+ </li>
<li class="active">
@@ -171,13 +185,13 @@
<i class="none"></i>
Management</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-parsers/index.html" title="Parsers">
- <i class="none"></i>
+ <i class="icon-chevron-right"></i>
Parsers</a>
- </li>
+ </li>
<li>
@@ -201,6 +215,20 @@
<i class="icon-chevron-right"></i>
Sensors</a>
</li>
+
+ <li>
+
+ <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
+ <i class="icon-chevron-right"></i>
+ Stellar-common</a>
+ </li>
+
+ <li>
+
+ <a href="../../use-cases/index.html" title="Use-cases">
+ <i class="icon-chevron-right"></i>
+ Use-cases</a>
+ </li>
</ul>
</li>
</ul>
@@ -338,6 +366,14 @@
<div class="section">
<h3><a name="STIX_Extractor"></a>STIX Extractor</h3>
<p>Consider the following config for importing STIX documents. This is a threat intelligence interchange format, so it is particularly relevant and attractive data to import for our purposes. Because STIX is a standard format, there is no need to specify the schema or how to interpret the documents.</p>
+<p>We support the versions of Stix and Cybox supported by <a class="externalLink" href="https://github.com/STIXProject/java-stix/tree/v1.2.0.2">java-stix</a>:</p>
+
+<ul>
+
+<li>Stix - <a class="externalLink" href="https://github.com/STIXProject/schemas/blob/356cc4f6b06625465f0808388eb166807313b4e0/stix_core.xsd">1.2</a> and earlier</li>
+
+<li>Cybox - <a class="externalLink" href="https://github.com/CybOXProject/schemas/blob/97beb32c376a9223e91b52cb3e4c8d2af6baf786/cybox_core.xsd">2.1</a> and earlier</li>
+</ul>
<p>We support a subset of STIX messages for importation:</p>
<table border="0" class="table table-striped">
@@ -407,6 +443,15 @@
<td>hostname </td>
</tr>
+
+<tr class="b">
+
+<td>URI </td>
+
+<td> </td>
+
+<td>uriobjecttype </td>
+ </tr>
</tbody>
</table>
<p>NOTE: The enrichment type will be used as the type above.</p>
http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-platform/metron-enrichment/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-enrichment/index.html b/site/current-book/metron-platform/metron-enrichment/index.html
index 771f646..979baf1 100644
--- a/site/current-book/metron-platform/metron-enrichment/index.html
+++ b/site/current-book/metron-platform/metron-enrichment/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-06-27
+ | Generated by Apache Maven Doxia at 2017-09-15
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170627" />
+ <meta name="Date-Revision-yyyymmdd" content="20170915" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Enrichment</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
@@ -61,8 +61,8 @@
- <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li>
- <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+ <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.1</li>
</ul>
</div>
@@ -75,7 +75,7 @@
<ul class="nav nav-list">
<li class="nav-header">User Documentation</li>
-
+
<li>
<a href="../../index.html" title="Metron">
@@ -96,7 +96,14 @@
<i class="icon-chevron-right"></i>
Analytics</a>
</li>
-
+
+ <li>
+
+ <a href="../../metron-contrib/metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
<li>
<a href="../../metron-deployment/index.html" title="Deployment">
@@ -106,9 +113,9 @@
<li>
- <a href="../../metron-docker/index.html" title="Docker">
+ <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
<i class="none"></i>
- Docker</a>
+ Alerts</a>
</li>
<li>
@@ -124,7 +131,7 @@
<i class="none"></i>
Rest</a>
</li>
-
+
<li>
<a href="../../metron-platform/index.html" title="Platform">
@@ -134,17 +141,24 @@
<li>
+ <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide">
+ <i class="none"></i>
+ Performance-tuning-guide</a>
+ </li>
+
+ <li>
+
<a href="../../metron-platform/metron-api/index.html" title="Api">
<i class="none"></i>
Api</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-common/index.html" title="Common">
- <i class="icon-chevron-right"></i>
+ <i class="none"></i>
Common</a>
- </li>
+ </li>
<li>
@@ -171,13 +185,13 @@
<i class="none"></i>
Management</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-parsers/index.html" title="Parsers">
- <i class="none"></i>
+ <i class="icon-chevron-right"></i>
Parsers</a>
- </li>
+ </li>
<li>
@@ -201,6 +215,20 @@
<i class="icon-chevron-right"></i>
Sensors</a>
</li>
+
+ <li>
+
+ <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
+ <i class="icon-chevron-right"></i>
+ Stellar-common</a>
+ </li>
+
+ <li>
+
+ <a href="../../use-cases/index.html" title="Use-cases">
+ <i class="icon-chevron-right"></i>
+ Use-cases</a>
+ </li>
</ul>
</li>
</ul>
@@ -535,6 +563,8 @@
<li><tt>MEAN</tt> : The mean of all of the associated values for matching queries</li>
+<li><tt>SUM</tt> : The sum of all the associated values for matching queries</li>
+
<li><tt>POSITIVE_MEAN</tt> : The mean of the positive associated values for the matching queries.</li>
</ul></div>
<div class="section">
@@ -691,33 +721,7 @@
<li>Ensure that the documents have new fields <tt>foo</tt>, <tt>bar</tt> and <tt>ALL_CAPS</tt> with values as described above.</li>
</ul>
-<p>Note that we could have used any Stellar statements here, including calling out to HBase via <tt>ENRICHMENT_GET</tt> and <tt>ENRICHMENT_EXISTS</tt> or even calling a machine learning model via <a href="../../metron-analytics/metron-maas-service/index.html">Model as a Service</a>.</p>
-<p><a name="Notes_on_Performance_Tuning"></a></p>
-<h1>Notes on Performance Tuning</h1>
-<p>Default installed Metron is untuned for production deployment. There are a few knobs to tune to get the most out of your system.</p></div>
-<div class="section">
-<h2><a name="Kafka_Queue"></a>Kafka Queue</h2>
-<p>The <tt>enrichments</tt> kafka queue is a collection point from all of the parser topologies. As such, make sure that the number of partitions in the kafka topic is sufficient to handle the throughput that you expect from your parser topologies.</p></div>
-<div class="section">
-<h2><a name="Enrichment_Topology"></a>Enrichment Topology</h2>
-<p>The enrichment topology as started by the <tt>$METRON_HOME/bin/start_enrichment_topology.sh</tt> script uses a default of one executor per bolt. In a real production system, this should be customized by modifying the flux file in <tt>$METRON_HOME/flux/enrichment/remote.yaml</tt>. </p>
-
-<ul>
-
-<li>Add a <tt>parallelism</tt> field to the bolts to give Storm a parallelism hint for the various components. Give bolts which appear to be bottlenecks (e.g. stellar enrichment bolt, hbase enrichment and threat intel bolts) a larger hint.</li>
-
-<li>Add a <tt>parallelism</tt> field to the kafka spout which matches the number of partitions for the enrichment kafka queue.</li>
-
-<li>Adjust the number of workers for the topology by adjusting the <tt>topology.workers</tt> field for the topology.</li>
-</ul>
-<p>Finally, if workers and executors are new to you or you don’t know where to modify the flux file, the following might be of use to you:</p>
-
-<ul>
-
-<li><a class="externalLink" href="http://www.michael-noll.com/blog/2012/10/16/understanding-the-parallelism-of-a-storm-topology/">Understanding the Parallelism of a Storm Topology</a></li>
-
-<li><a class="externalLink" href="http://storm.apache.org/releases/current/flux.html">Flux Docs</a></li>
-</ul></div>
+<p>Note that we could have used any Stellar statements here, including calling out to HBase via <tt>ENRICHMENT_GET</tt> and <tt>ENRICHMENT_EXISTS</tt> or even calling a machine learning model via <a href="../../metron-analytics/metron-maas-service/index.html">Model as a Service</a>.</p></div>
</div>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-platform/metron-indexing/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-indexing/index.html b/site/current-book/metron-platform/metron-indexing/index.html
index febd70e..fc5fef2 100644
--- a/site/current-book/metron-platform/metron-indexing/index.html
+++ b/site/current-book/metron-platform/metron-indexing/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-06-27
+ | Generated by Apache Maven Doxia at 2017-09-15
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170627" />
+ <meta name="Date-Revision-yyyymmdd" content="20170915" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Indexing</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
@@ -61,8 +61,8 @@
- <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li>
- <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+ <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.1</li>
</ul>
</div>
@@ -75,7 +75,7 @@
<ul class="nav nav-list">
<li class="nav-header">User Documentation</li>
-
+
<li>
<a href="../../index.html" title="Metron">
@@ -96,7 +96,14 @@
<i class="icon-chevron-right"></i>
Analytics</a>
</li>
-
+
+ <li>
+
+ <a href="../../metron-contrib/metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
<li>
<a href="../../metron-deployment/index.html" title="Deployment">
@@ -106,9 +113,9 @@
<li>
- <a href="../../metron-docker/index.html" title="Docker">
+ <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
<i class="none"></i>
- Docker</a>
+ Alerts</a>
</li>
<li>
@@ -124,7 +131,7 @@
<i class="none"></i>
Rest</a>
</li>
-
+
<li>
<a href="../../metron-platform/index.html" title="Platform">
@@ -134,17 +141,24 @@
<li>
+ <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide">
+ <i class="none"></i>
+ Performance-tuning-guide</a>
+ </li>
+
+ <li>
+
<a href="../../metron-platform/metron-api/index.html" title="Api">
<i class="none"></i>
Api</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-common/index.html" title="Common">
- <i class="icon-chevron-right"></i>
+ <i class="none"></i>
Common</a>
- </li>
+ </li>
<li>
@@ -171,13 +185,13 @@
<i class="none"></i>
Management</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-parsers/index.html" title="Parsers">
- <i class="none"></i>
+ <i class="icon-chevron-right"></i>
Parsers</a>
- </li>
+ </li>
<li>
@@ -201,6 +215,20 @@
<i class="icon-chevron-right"></i>
Sensors</a>
</li>
+
+ <li>
+
+ <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
+ <i class="icon-chevron-right"></i>
+ Stellar-common</a>
+ </li>
+
+ <li>
+
+ <a href="../../use-cases/index.html" title="Use-cases">
+ <i class="icon-chevron-right"></i>
+ Use-cases</a>
+ </li>
</ul>
</li>
</ul>
@@ -238,7 +266,7 @@
<li>Solr</li>
</ul>
<p>By default, this topology writes out to both HDFS and one of Elasticsearch and Solr.</p>
-<p>Indices are written in batch and the batch size is specified in the <a href="#Sensor_Indexing_Configuration">Sensor Indexing Configuration</a> via the <tt>batchSize</tt> parameter. This config is variable by sensor type.</p></div>
+<p>Indices are written in batch and the batch size and batch timeout are specified in the <a href="#Sensor_Indexing_Configuration">Sensor Indexing Configuration</a> via the <tt>batchSize</tt> and <tt>batchTimeout</tt> parameters. These configs are variable by sensor type.</p></div>
<div class="section">
<h2><a name="Indexing_Architecture"></a>Indexing Architecture</h2>
<p><img src="../../images/indexing_arch.png" alt="Architecture" /></p>
@@ -271,7 +299,9 @@
<li><tt>index</tt> : The name of the index to write to (defaulted to the name of the sensor).</li>
-<li><tt>batchSize</tt> : The size of the batch that is written to the indices at once (defaulted to <tt>1</tt>).</li>
+<li><tt>batchSize</tt> : The size of the batch that is written to the indices at once. Defaults to <tt>1</tt> (no batching).</li>
+
+<li><tt>batchTimeout</tt> : The timeout after which a batch will be flushed even if batchSize has not been met. Optional. If unspecified, or set to <tt>0</tt>, it defaults to a system-determined duration which is a fraction of the Storm parameter <tt>topology.message.timeout.secs</tt>. Ignored if batchSize is <tt>1</tt>, since this disables batching.</li>
<li><tt>enabled</tt> : Whether the writer is enabled (default <tt>true</tt>).</li>
</ul>
@@ -298,6 +328,8 @@
<li>batch size of 1</li>
+<li>batch timeout system default</li>
+
<li>index name the same as the sensor</li>
</ul></li>
@@ -309,6 +341,8 @@
<li>batch size of 1</li>
+<li>batch timeout system default</li>
+
<li>index name the same as the sensor</li>
</ul></li>
</ul>
@@ -322,11 +356,13 @@
"elasticsearch": {
"index": "foo",
"batchSize" : 100,
+ "batchTimeout" : 0,
"enabled" : true
},
"hdfs": {
"index": "foo",
"batchSize": 1,
+ "batchTimeout" : 0,
"enabled" : true
}
}
@@ -342,6 +378,8 @@
<li>batch size of 100</li>
+<li>batch timeout system default</li>
+
<li>index name of “foo”</li>
</ul></li>
@@ -353,6 +391,8 @@
<li>batch size of 1</li>
+<li>batch timeout system default</li>
+
<li>index name of “foo”</li>
</ul></li>
</ul></div>
@@ -369,6 +409,7 @@
"hdfs": {
"index": "foo",
"batchSize": 100,
+ "batchTimeout" : 0,
"enabled" : false
}
}
@@ -384,6 +425,8 @@
<li>batch size of 1</li>
+<li>batch timeout system default</li>
+
<li>index name of “foo”</li>
</ul></li>
@@ -394,9 +437,24 @@
<li>disabled</li>
</ul></li>
</ul>
+<p><a name="Updates_to_Indexed_Data"></a></p>
+<h1>Updates to Indexed Data</h1>
+<p>There are clear usecases where we would want to incorporate the capability to update indexed data. Thus far, we have limited capabilities provided to support this use-case:</p>
+
+<ul>
+
+<li>Updates to the random access index (e.g. Elasticsearch and Solr) should be supported</li>
+
+<li>Updates to the cold storage index (e.g. HDFS) is not supported currently, however to support the batch use-case updated documents will be provided in a NoSQL write-ahead log (e.g. a HBase table) and an Java API will be provided to retrieve those updates scalably (i.e. a scan-free architecture).</li>
+</ul>
+<p>Put simply, the random access index will be always up-to-date, but the HDFS index will need to be joined to the NoSQL write-ahead log to get current updates.</p></div></div></div>
+<div class="section">
+<h2><a name="The_IndexDao_Abstraction"></a>The <tt>IndexDao</tt> Abstraction</h2>
+<p>The indices mentioned above as part of Update should be pluggable by the developer so that new write-ahead logs or real-time indices can be supported by providing an implementation supporting the data access patterns.</p>
+<p>To support a new index, one would need to implement the <tt>org.apache.metron.indexing.dao.IndexDao</tt> abstraction and provide update and search capabilities. IndexDaos may be composed and updates will be performed in parallel. This enables a flexible strategy for specifying your backing store for updates at runtime. For instance, currently the REST API supports the update functionality and may be configured with a list of IndexDao implementations to use to support the updates.</p>
<p><a name="Notes_on_Performance_Tuning"></a></p>
<h1>Notes on Performance Tuning</h1>
-<p>Default installed Metron is untuned for production deployment. By far and wide, the most likely piece to require TLC from a performance perspective is the indexing layer. An index that does not keep up will back up and you will see errors in the kafka bolt. There are a few knobs to tune to get the most out of your system.</p></div></div></div>
+<p>Default installed Metron is untuned for production deployment. By far and wide, the most likely piece to require TLC from a performance perspective is the indexing layer. An index that does not keep up will back up and you will see errors in the kafka bolt. There are a few knobs to tune to get the most out of your system.</p></div>
<div class="section">
<h2><a name="Kafka_Queue"></a>Kafka Queue</h2>
<p>The <tt>indexing</tt> kafka queue is a collection point from the enrichment topology. As such, make sure that the number of partitions in the kafka topic is sufficient to handle the throughput that you expect.</p></div>
http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-platform/metron-management/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-management/index.html b/site/current-book/metron-platform/metron-management/index.html
index 6efed91..a760baf 100644
--- a/site/current-book/metron-platform/metron-management/index.html
+++ b/site/current-book/metron-platform/metron-management/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-06-27
+ | Generated by Apache Maven Doxia at 2017-09-15
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170627" />
+ <meta name="Date-Revision-yyyymmdd" content="20170915" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Stellar REPL Management Utilities</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
@@ -61,8 +61,8 @@
- <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li>
- <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+ <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.1</li>
</ul>
</div>
@@ -75,7 +75,7 @@
<ul class="nav nav-list">
<li class="nav-header">User Documentation</li>
-
+
<li>
<a href="../../index.html" title="Metron">
@@ -96,7 +96,14 @@
<i class="icon-chevron-right"></i>
Analytics</a>
</li>
-
+
+ <li>
+
+ <a href="../../metron-contrib/metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
<li>
<a href="../../metron-deployment/index.html" title="Deployment">
@@ -106,9 +113,9 @@
<li>
- <a href="../../metron-docker/index.html" title="Docker">
+ <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
<i class="none"></i>
- Docker</a>
+ Alerts</a>
</li>
<li>
@@ -124,7 +131,7 @@
<i class="none"></i>
Rest</a>
</li>
-
+
<li>
<a href="../../metron-platform/index.html" title="Platform">
@@ -134,17 +141,24 @@
<li>
+ <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide">
+ <i class="none"></i>
+ Performance-tuning-guide</a>
+ </li>
+
+ <li>
+
<a href="../../metron-platform/metron-api/index.html" title="Api">
<i class="none"></i>
Api</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-common/index.html" title="Common">
- <i class="icon-chevron-right"></i>
+ <i class="none"></i>
Common</a>
- </li>
+ </li>
<li>
@@ -171,13 +185,13 @@
<a href="#"><i class="none"></i>Management</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-parsers/index.html" title="Parsers">
- <i class="none"></i>
+ <i class="icon-chevron-right"></i>
Parsers</a>
- </li>
+ </li>
<li>
@@ -201,6 +215,20 @@
<i class="icon-chevron-right"></i>
Sensors</a>
</li>
+
+ <li>
+
+ <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
+ <i class="icon-chevron-right"></i>
+ Stellar-common</a>
+ </li>
+
+ <li>
+
+ <a href="../../use-cases/index.html" title="Use-cases">
+ <i class="icon-chevron-right"></i>
+ Use-cases</a>
+ </li>
</ul>
</li>
</ul>
@@ -260,7 +288,7 @@
<li>Parser functions - Functions surrounding adding, viewing, and removing Parser functions.</li>
-<li>Enrichment functions - Functions surrounding adding, viewing and removing Stellar enrichments as well as managing batch size and index names for the enrichment topology configuration</li>
+<li>Enrichment functions - Functions surrounding adding, viewing and removing Stellar enrichments as well as managing batch size, batch timeout, and index names for the enrichment topology configuration</li>
<li>Threat Triage functions - Functions surrounding adding, viewing and removing threat triage functions.</li>
</ul>
@@ -682,7 +710,7 @@
<ul>
-<li>Description: Set batch size</li>
+<li>Description: Set batch size and timeout</li>
<li>Input:
@@ -693,6 +721,8 @@
<li>writer - The writer to update (e.g. elasticsearch, solr or hdfs)</li>
<li>size - batch size (integer), defaults to 1, meaning batching disabled</li>
+
+<li>timeout - (optional) batch timeout in seconds (integer), defaults to 0, meaning system default</li>
</ul></li>
<li>Returns: The String representation of the config in zookeeper</li>
@@ -886,7 +916,23 @@
</ul></div></div>
<div class="section">
<h2><a name="Deployment_Instructions"></a>Deployment Instructions</h2>
-<p>Deployment is as simple as dropping the jar created by this project into <tt>$METRON_HOME/lib</tt> and starting the Stellar shell via <tt>$METRON_HOME/bin/stellar</tt></p></div>
+
+<ul>
+
+<li>Clusters installed via Ambari Management Pack (default)
+
+<ul>
+
+<li>Automatically deployed</li>
+ </ul></li>
+
+<li>Manual installation
+
+<ul>
+
+<li>Deployment is as simple as dropping the jar created by this project into <tt>$METRON_HOME/lib</tt> and starting the Stellar shell via <tt>$METRON_HOME/bin/stellar</tt></li>
+ </ul></li>
+</ul></div>
<div class="section">
<h2><a name="Examples"></a>Examples</h2>
<p>Included for description and education purposes are a couple example Stellar REPL transcripts with helpful comments to illustrate some common operations.</p>
http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-platform/metron-parsers/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-parsers/index.html b/site/current-book/metron-platform/metron-parsers/index.html
index f7d13a6..529badc 100644
--- a/site/current-book/metron-platform/metron-parsers/index.html
+++ b/site/current-book/metron-platform/metron-parsers/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-06-27
+ | Generated by Apache Maven Doxia at 2017-09-15
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170627" />
+ <meta name="Date-Revision-yyyymmdd" content="20170915" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Parsers</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
@@ -61,8 +61,8 @@
- <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li>
- <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+ <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.1</li>
</ul>
</div>
@@ -75,7 +75,7 @@
<ul class="nav nav-list">
<li class="nav-header">User Documentation</li>
-
+
<li>
<a href="../../index.html" title="Metron">
@@ -96,7 +96,14 @@
<i class="icon-chevron-right"></i>
Analytics</a>
</li>
-
+
+ <li>
+
+ <a href="../../metron-contrib/metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
<li>
<a href="../../metron-deployment/index.html" title="Deployment">
@@ -106,9 +113,9 @@
<li>
- <a href="../../metron-docker/index.html" title="Docker">
+ <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
<i class="none"></i>
- Docker</a>
+ Alerts</a>
</li>
<li>
@@ -124,7 +131,7 @@
<i class="none"></i>
Rest</a>
</li>
-
+
<li>
<a href="../../metron-platform/index.html" title="Platform">
@@ -134,17 +141,24 @@
<li>
+ <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide">
+ <i class="none"></i>
+ Performance-tuning-guide</a>
+ </li>
+
+ <li>
+
<a href="../../metron-platform/metron-api/index.html" title="Api">
<i class="none"></i>
Api</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-common/index.html" title="Common">
- <i class="icon-chevron-right"></i>
+ <i class="none"></i>
Common</a>
- </li>
+ </li>
<li>
@@ -173,11 +187,20 @@
<i class="none"></i>
Management</a>
</li>
-
+
<li class="active">
- <a href="#"><i class="none"></i>Parsers</a>
- </li>
+ <a href="#"><i class="icon-chevron-down"></i>Parsers</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../metron-platform/metron-parsers/parser-testing.html" title="parser-testing">
+ <i class="none"></i>
+ parser-testing</a>
+ </li>
+ </ul>
+ </li>
<li>
@@ -201,6 +224,20 @@
<i class="icon-chevron-right"></i>
Sensors</a>
</li>
+
+ <li>
+
+ <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
+ <i class="icon-chevron-right"></i>
+ Stellar-common</a>
+ </li>
+
+ <li>
+
+ <a href="../../use-cases/index.html" title="Use-cases">
+ <i class="icon-chevron-right"></i>
+ Use-cases</a>
+ </li>
</ul>
</li>
</ul>
@@ -263,7 +300,27 @@
<li><tt>columns</tt> : A map of column names you wish to extract from the CSV to their offsets (e.g. <tt>{ 'name' : 1, 'profession' : 3}</tt> would be a column map for extracting the 2nd and 4th columns from a CSV)</li>
-<li><tt>separator</tt> : The column separator, <tt>,</tt> by default. just</li>
+<li><tt>separator</tt> : The column separator, <tt>,</tt> by default.</li>
+ </ul></li>
+
+<li>JSON Map Parser: <tt>org.apache.metron.parsers.json.JSONMapParser</tt> with possible <tt>parserConfig</tt> entries of
+
+<ul>
+
+<li><tt>mapStrategy</tt> : A strategy to indicate how to handle multi-dimensional Maps. This is one of
+
+<ul>
+
+<li><tt>DROP</tt> : Drop fields which contain maps</li>
+
+<li><tt>UNFOLD</tt> : Unfold inner maps. So <tt>{ "foo" : { "bar" : 1} }</tt> would turn into <tt>{"foo.bar" : 1}</tt></li>
+
+<li><tt>ALLOW</tt> : Allow multidimensional maps</li>
+
+<li><tt>ERROR</tt> : Throw an error when a multidimensional map is encountered</li>
+ </ul></li>
+
+<li>A field called <tt>timestamp</tt> is expected to exist and, if it does not, then current time is inserted.</li>
</ul></li>
</ul></li>
</ul></div>
@@ -346,11 +403,37 @@
<ul>
-<li><tt>sensorTopic</tt> : The kafka topic to send the parsed messages to.</li>
+<li><tt>sensorTopic</tt> : The kafka topic to send the parsed messages to. If the topic is prefixed and suffixed by <tt>/</tt> then it is assumed to be a regex and will match any topic matching the pattern (e.g. <tt>/bro.*/</tt> would match <tt>bro_cust0</tt>, <tt>bro_cust1</tt> and <tt>bro_cust2</tt>)</li>
+
+<li><tt>readMetadata</tt> : Boolean indicating whether to read metadata or not (<tt>false</tt> by default). See below for a discussion about metadata.</li>
+
+<li><tt>mergeMetadata</tt> : Boolean indicating whether to merge metadata with the message or not (<tt>false</tt> by default). See below for a discussion about metadata.</li>
<li><tt>parserConfig</tt> : A JSON Map representing the parser implementation specific configuration.</li>
<li><tt>fieldTransformations</tt> : An array of complex objects representing the transformations to be done on the message generated from the parser before writing out to the kafka topic.</li>
+
+<li><tt>spoutParallelism</tt> : The kafka spout parallelism (default to <tt>1</tt>). This can be overridden on the command line.</li>
+
+<li><tt>spoutNumTasks</tt> : The number of tasks for the spout (default to <tt>1</tt>). This can be overridden on the command line.</li>
+
+<li><tt>parserParallelism</tt> : The parser bolt parallelism (default to <tt>1</tt>). This can be overridden on the command line.</li>
+
+<li><tt>parserNumTasks</tt> : The number of tasks for the parser bolt (default to <tt>1</tt>). This can be overridden on the command line.</li>
+
+<li><tt>errorWriterParallelism</tt> : The error writer bolt parallelism (default to <tt>1</tt>). This can be overridden on the command line.</li>
+
+<li><tt>errorWriterNumTasks</tt> : The number of tasks for the error writer bolt (default to <tt>1</tt>). This can be overridden on the command line.</li>
+
+<li><tt>numWorkers</tt> : The number of workers to use in the topology (default is the storm default of <tt>1</tt>).</li>
+
+<li><tt>numAckers</tt> : The number of acker executors to use in the topology (default is the storm default of <tt>1</tt>).</li>
+
+<li><tt>spoutConfig</tt> : A map representing a custom spout config (this is a map). This can be overridden on the command line.</li>
+
+<li><tt>securityProtocol</tt> : The security protocol to use for reading from kafka (this is a string). This can be overridden on the command line and also specified in the spout config via the <tt>security.protocol</tt> key. If both are specified, then they are merged and the CLI will take precedence.</li>
+
+<li><tt>stormConfig</tt> : The storm config to use (this is a map). This can be overridden on the command line. If both are specified, they are merged with CLI properties taking precedence.</li>
</ul>
<p>The <tt>fieldTransformations</tt> is a complex object which defines a transformation which can be done to a message. This transformation can </p>
@@ -363,6 +446,59 @@
<li>Remove existing fields of a message</li>
</ul>
<div class="section">
+<h3><a name="Metadata"></a>Metadata</h3>
+<p>Metadata is a useful thing to send to Metron and use during enrichment or threat intelligence.<br />Consider the following scenarios:</p>
+
+<ul>
+
+<li>You have multiple telemetry sources of the same type that you want to
+
+<ul>
+
+<li>ensure downstream analysts can differentiate</li>
+
+<li>ensure profiles consider independently as they have different seasonality or some other fundamental characteristic</li>
+ </ul></li>
+</ul>
+<p>As such, there are two types of metadata that we seek to support in Metron:</p>
+
+<ul>
+
+<li>Environmental metadata : Metadata about the system at large
+
+<ul>
+
+<li>Consider the possibility that you have multiple kafka topics being processed by one parser and you want to tag the messages with the kafka topic</li>
+
+<li>At the moment, only the kafka topic is kept as the field name.</li>
+ </ul></li>
+
+<li>Custom metadata: Custom metadata from an individual telemetry source that one might want to use within Metron.</li>
+</ul>
+<p>Metadata is controlled by two fields in the parser:</p>
+
+<ul>
+
+<li><tt>readMetadata</tt> : This is a boolean indicating whether metadata will be read and made available to Field transformations (i.e. Stellar field transformations). The default is <tt>false</tt>.</li>
+
+<li>
+<p><tt>mergeMetadata</tt> : This is a boolean indicating whether metadata fields will be merged with the message automatically.<br />That is to say, if this property is set to <tt>true</tt> then every metadata field will become part of the messages and, consequently, also available for use in field transformations.</p>
+<div class="section">
+<h4><a name="Field_Naming"></a>Field Naming</h4></li>
+</ul>
+<p>In order to avoid collisions from metadata fields, metadata fields will be prefixed with <tt>metron.metadata.</tt>.<br />So, for instance the kafka topic would be in the field <tt>metron.metadata.topic</tt>.</p></div>
+<div class="section">
+<h4><a name="Specifying_Custom_Metadata"></a>Specifying Custom Metadata</h4>
+<p>Custom metadata is specified by sending a JSON Map in the key. If no key is sent, then, obviously, no metadata will be parsed. For instance, sending a metadata field called <tt>customer_id</tt> could be done by sending</p>
+
+<div class="source">
+<div class="source">
+<pre>{
+"customer_id" : "my_customer_id"
+}
+</pre></div></div>
+<p>in the kafka key. This would be exposed as the field <tt>metron.metadata.customer_id</tt> to stellar field transformations as well, if <tt>mergeMetadata</tt> is <tt>true</tt>, available as a field in its own right.</p></div></div>
+<div class="section">
<h3><a name="fieldTransformation_configuration"></a><tt>fieldTransformation</tt> configuration</h3>
<p>The format of a <tt>fieldTransformation</tt> is as follows:</p>
@@ -437,7 +573,45 @@
<ul>
<li><tt>STELLAR</tt> : This transformation executes a set of transformations expressed as <a href="../metron-common/index.html">Stellar Language</a> statements.</li>
-</ul>
+</ul></div>
+<div class="section">
+<h3><a name="Assignment_to_null"></a>Assignment to <tt>null</tt></h3>
+<p>If, in your field transformation, you assign a field to <tt>null</tt>, the field will be removed. You can use this capability to rename variables.</p>
+<p>Consider this example:</p>
+
+<div class="source">
+<div class="source">
+<pre> "fieldTransformations" : [
+ { "transformation" : "STELLAR"
+ ,"output" : [ "new_field", "old_field"]
+ ,"config" : {
+ "new_field" : "old_field"
+ ,"old_field" : "null"
+ }
+ }
+ ]
+</pre></div></div>
+<p>This would set <tt>new_field</tt> to the value of <tt>old_field</tt> and remove <tt>old_field</tt>.</p></div>
+<div class="section">
+<h3><a name="Warning:_Transforming_the_same_field_twice"></a>Warning: Transforming the same field twice</h3>
+<p>Currently, the stellar expressions are expressed in the form of a map where the keys define the fields and the values define the Stellar expressions. You order the expression evaluation in the <tt>output</tt> field. A consequence of this choice to store the assignments as a map is that the same field cannot appear in the map as a key twice.</p>
+<p>For instance, the following will not function as expected:</p>
+
+<div class="source">
+<div class="source">
+<pre> "fieldTransformations" : [
+ { "transformation" : "STELLAR"
+ ,"output" : [ "new_field"]
+ ,"config" : {
+ "new_field" : "TO_UPPER(field1)"
+ ,"new_field" : "TO_LOWER(new_field)"
+ }
+ }
+ ]
+</pre></div></div>
+<p>In the above example, the last instance of <tt>new_field</tt> will win and <tt>TO_LOWER(new_field)</tt> will be evaluated while <tt>TO_UPPER(field1)</tt> will be skipped.</p></div>
+<div class="section">
+<h3><a name="Example"></a>Example</h3>
<p>Consider the following sensor parser config to add three new fields to a message:</p>
<ul>
http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-platform/metron-parsers/parser-testing.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-parsers/parser-testing.html b/site/current-book/metron-platform/metron-parsers/parser-testing.html
new file mode 100644
index 0000000..25288a3
--- /dev/null
+++ b/site/current-book/metron-platform/metron-parsers/parser-testing.html
@@ -0,0 +1,372 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-09-15
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20170915" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Parser Contribution and Testing</title>
+ <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../../css/site.css" />
+ <link rel="stylesheet" href="../../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+
+ </head>
+ <body class="topBarDisabled">
+
+
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="http://metron.apache.org/" id="bannerLeft">
+ <img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org" class="externalLink" title="Apache">
+ Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+ Metron</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="../../index.html" title="Documentation">
+ Documentation</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Parser Contribution and Testing</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.1</li>
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span3">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+
+ <li>
+
+ <a href="../../index.html" title="Metron">
+ <i class="icon-chevron-down"></i>
+ Metron</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../Upgrading.html" title="Upgrading">
+ <i class="none"></i>
+ Upgrading</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-analytics/index.html" title="Analytics">
+ <i class="icon-chevron-right"></i>
+ Analytics</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-contrib/metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-deployment/index.html" title="Deployment">
+ <i class="icon-chevron-right"></i>
+ Deployment</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
+ <i class="none"></i>
+ Alerts</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-interface/metron-config/index.html" title="Config">
+ <i class="none"></i>
+ Config</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-interface/metron-rest/index.html" title="Rest">
+ <i class="none"></i>
+ Rest</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/index.html" title="Platform">
+ <i class="icon-chevron-down"></i>
+ Platform</a>
+ <ul class="nav nav-list">
+
+ <li>
+
+ <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide">
+ <i class="none"></i>
+ Performance-tuning-guide</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/metron-api/index.html" title="Api">
+ <i class="none"></i>
+ Api</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/metron-common/index.html" title="Common">
+ <i class="none"></i>
+ Common</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/metron-data-management/index.html" title="Data-management">
+ <i class="none"></i>
+ Data-management</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment">
+ <i class="none"></i>
+ Enrichment</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/metron-indexing/index.html" title="Indexing">
+ <i class="none"></i>
+ Indexing</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/metron-management/index.html" title="Management">
+ <i class="none"></i>
+ Management</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/metron-parsers/index.html" title="Parsers">
+ <i class="icon-chevron-down"></i>
+ Parsers</a>
+ <ul class="nav nav-list">
+
+ <li class="active">
+
+ <a href="#"><i class="none"></i>parser-testing</a>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend">
+ <i class="none"></i>
+ Pcap-backend</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-platform/metron-writer/index.html" title="Writer">
+ <i class="none"></i>
+ Writer</a>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+
+ <a href="../../metron-sensors/index.html" title="Sensors">
+ <i class="icon-chevron-right"></i>
+ Sensors</a>
+ </li>
+
+ <li>
+
+ <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
+ <i class="icon-chevron-right"></i>
+ Stellar-common</a>
+ </li>
+
+ <li>
+
+ <a href="../../use-cases/index.html" title="Use-cases">
+ <i class="icon-chevron-right"></i>
+ Use-cases</a>
+ </li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span9" >
+
+ <h1>Parser Contribution and Testing</h1>
+<p><a name="Parser_Contribution_and_Testing"></a></p>
+<p>So you want to contribute a parser to Apache Metron. First off, on behalf of the community, thank you very much! Now that you have implemented a parser by writing a java class which implements <tt>org.apache.metron.parsers.interfaces.MessageParser</tt> what are the testing expectations for a new parser?</p>
+<p>It is expected that a new parser have two tests:</p>
+
+<ul>
+
+<li>A JUnit test directly testing your parser class.</li>
+
+<li>An Integration test validating that your parser class can parse messages inside the <tt>ParserBolt</tt>.</li>
+</ul>
+<div class="section">
+<h2><a name="The_JUnit_Test"></a>The JUnit Test</h2>
+<p>The JUnit Test should be focused on testing your Parser directly. You should feel free to use mocks or stubs or whatever else you need to completely test that unit of functionality.</p></div>
+<div class="section">
+<h2><a name="The_Integration_Test"></a>The Integration Test</h2>
+<p>Integration tests are more structured. The intent is that the parser that you have implemented can be driven successfully from <tt>org.apache.metron.parsers.bolt.ParserBolt</tt>.</p>
+<p>The procedure for creating a new test is as follows:</p>
+
+<ul>
+
+<li>Create an integration test that extends <tt>org.apache.metron.parsers.integration.ParserIntegrationTest</tt>
+
+<ul>
+
+<li>Override <tt>getSensorType()</tt> to return the sensor type to be used in the test (referred to as <tt>${sensor_type}</tt> at times)</li>
+
+<li>Override <tt>getValidations()</tt> to indicate how you want the output of the parser to be validated (more on validations later)</li>
+
+<li>Optionally <tt>readSensorConfig(String sensorType)</tt> to read the sensor config
+
+<ul>
+
+<li>By default, we will pull this from <tt>metron-parsers/src/main/config/zookeeper/parsers/${sensor_type}</tt>. Override if you want to provide your own</li>
+ </ul></li>
+
+<li>Optionally <tt>readGlobalConfig()</tt> to return the global config
+
+<ul>
+
+<li>By default, we will pull this from <tt>metron-integration-test/src/main/config/zookeeper/global.json)</tt>. Override if you want to provide your own</li>
+ </ul></li>
+ </ul></li>
+
+<li>Place sample input data in <tt>metron-integration-test/src/main/sample/data/${sensor_type}/raw</tt>
+
+<ul>
+
+<li>It should be one line per input record.</li>
+ </ul></li>
+
+<li>Place expected output based on sample data in <tt>metron-integration-test/src/main/sample/data/${sensor_type}/parsed</tt>
+
+<ul>
+
+<li>Line <tt>k</tt> in the expected data should match with line <tt>k</tt></li>
+ </ul></li>
+</ul>
+<p>The way these tests function is by creating a <tt>ParserBolt</tt> instance with your specified global configuration and sensor configuration. It will then send your specified sample input data in line-by-line. It will then perform some basic sanity validation:</p>
+
+<ul>
+
+<li>Ensure no errors were logged</li>
+
+<li>Execute your specified validation methods</li>
+</ul>
+<div class="section">
+<h3><a name="Validations"></a>Validations</h3>
+<p>Validations are functions which indicate how one should validate the parsed messages. The basic one which is sufficient for most cases is <tt>org.apache.metron.parsers.integration.validation.SampleDataValidation</tt>. This will read the expected results from <tt>metron-integration-test/src/main/sample/data/${sensor_type}/parsed</tt> and validate that the actual parsed message conforms (excluding timestamp).</p>
+<p>If you have special validations required, you may implement your own and return an instance of that in the <tt>getValidations()</tt> method of your Integration Test.</p></div>
+<div class="section">
+<h3><a name="Sample_Integration_Test"></a>Sample Integration Test</h3>
+<p>A sample integration test for the <tt>snort</tt> parser is as follows:</p>
+
+<div class="source">
+<div class="source">
+<pre>public class SnortIntegrationTest extends ParserIntegrationTest {
+ @Override
+ String getSensorType() {
+ return "snort";
+ }
+
+ @Override
+ List<ParserValidation> getValidations() {
+ return new ArrayList<ParserValidation>() {{
+ add(new SampleDataValidation());
+ }};
+ }
+}
+</pre></div></div></div></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2017
+ <a href="https://www.apache.org">The Apache Software Foundation</a>.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/metron/blob/87ff7b73/site/current-book/metron-platform/metron-pcap-backend/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-pcap-backend/index.html b/site/current-book/metron-platform/metron-pcap-backend/index.html
index af673d5..601fac8 100644
--- a/site/current-book/metron-platform/metron-pcap-backend/index.html
+++ b/site/current-book/metron-platform/metron-pcap-backend/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia at 2017-06-27
+ | Generated by Apache Maven Doxia at 2017-09-15
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20170627" />
+ <meta name="Date-Revision-yyyymmdd" content="20170915" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron PCAP Backend</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
@@ -61,8 +61,8 @@
- <li id="publishDate" class="pull-right">Last Published: 2017-06-27</li> <li class="divider pull-right">|</li>
- <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+ <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.4.1</li>
</ul>
</div>
@@ -75,7 +75,7 @@
<ul class="nav nav-list">
<li class="nav-header">User Documentation</li>
-
+
<li>
<a href="../../index.html" title="Metron">
@@ -96,7 +96,14 @@
<i class="icon-chevron-right"></i>
Analytics</a>
</li>
-
+
+ <li>
+
+ <a href="../../metron-contrib/metron-docker/index.html" title="Docker">
+ <i class="none"></i>
+ Docker</a>
+ </li>
+
<li>
<a href="../../metron-deployment/index.html" title="Deployment">
@@ -106,9 +113,9 @@
<li>
- <a href="../../metron-docker/index.html" title="Docker">
+ <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
<i class="none"></i>
- Docker</a>
+ Alerts</a>
</li>
<li>
@@ -124,7 +131,7 @@
<i class="none"></i>
Rest</a>
</li>
-
+
<li>
<a href="../../metron-platform/index.html" title="Platform">
@@ -134,17 +141,24 @@
<li>
+ <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide">
+ <i class="none"></i>
+ Performance-tuning-guide</a>
+ </li>
+
+ <li>
+
<a href="../../metron-platform/metron-api/index.html" title="Api">
<i class="none"></i>
Api</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-common/index.html" title="Common">
- <i class="icon-chevron-right"></i>
+ <i class="none"></i>
Common</a>
- </li>
+ </li>
<li>
@@ -173,13 +187,13 @@
<i class="none"></i>
Management</a>
</li>
-
+
<li>
<a href="../../metron-platform/metron-parsers/index.html" title="Parsers">
- <i class="none"></i>
+ <i class="icon-chevron-right"></i>
Parsers</a>
- </li>
+ </li>
<li class="active">
@@ -201,6 +215,20 @@
<i class="icon-chevron-right"></i>
Sensors</a>
</li>
+
+ <li>
+
+ <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
+ <i class="icon-chevron-right"></i>
+ Stellar-common</a>
+ </li>
+
+ <li>
+
+ <a href="../../use-cases/index.html" title="Use-cases">
+ <i class="icon-chevron-right"></i>
+ Use-cases</a>
+ </li>
</ul>
</li>
</ul>